chris/pixelfed
1
0
Fork 0
forked from mirror/pixelfed
Code Pull requests Activity

Isert cities in chunks to increase performance

Browse source
This commit is contained in:
monkeyless 2019-08-09 05:38:48 +02:00
parent 450492db53
commit 27952883b6
1 changed files with 25 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
app/Console/Commands/ImportCities.php
View file

@ -1,5 +1,4 @@
<?php <?php
namespace App\Console\Commands; namespace App\Console\Commands;
use Illuminate\Console\Command; use Illuminate\Console\Command;
@ -14,15 +13,13 @@ class ImportCities extends Command
* *
* @var string * @var string
*/ */
protected $signature = 'import:cities'; protected $signature = 'import:cities {chunk=1000}';
/** /**
* The console command description. * The console command description.
* *
* @var string * @var string
*/ */
protected $description = 'Import Cities to database'; protected $description = 'Import Cities to database';
/** /**
* Create a new command instance. * Create a new command instance.
* *
@ -32,7 +29,6 @@ class ImportCities extends Command
{ {
parent::__construct(); parent::__construct();
} }
/** /**
* Execute the console command. * Execute the console command.
* *
@ -45,34 +41,37 @@ class ImportCities extends Command
$this->error('Missing storage/app/cities.json file!'); $this->error('Missing storage/app/cities.json file!');
return; return;
} }
// if (Place::count() > 10) {
if(Place::count() > 10) { // $this->error('Cities already imported, aborting operation...');
$this->error('Cities already imported, aborting operation...'); // return;
return; // }
}
$this->info('Importing city data into database ...'); $this->info('Importing city data into database ...');
$cities = file_get_contents($path); $cities = file_get_contents($path);
$cities = json_decode($cities); $cities = json_decode($cities);
$count = count($cities); $cityCount = count($cities);
$this->info("Found {$count} cities to insert ..."); $this->info("Found {$cityCount} cities to insert ...");
$bar = $this->output->createProgressBar($count); $bar = $this->output->createProgressBar($cityCount);
$bar->start(); $bar->start();
$buffer = [];
$count = 0;
foreach ($cities as $city) { foreach ($cities as $city) {
$country = $city->country == 'XK' ? 'Kosovo' : (new \League\ISO3166\ISO3166)->alpha2($city->country)['name']; $country = $city->country == 'XK' ? 'Kosovo' : (new \League\ISO3166\ISO3166)->alpha2($city->country)['name'];
DB::transaction(function () use ($city, $country) { $buffer[] = ["name" => $city->name, "slug" => Str::slug($city->name), "country" => $country, "lat" => $city->lat, "long" => $city->lng];
$place = new Place(); $count++;
$place->name = $city->name; if ($count % $this->argument('chunk') == 0) {
$place->slug = Str::slug($city->name); $this->insertBuffer($buffer, $count);
$place->country = $country; $bar->advance(count($buffer));
$place->lat = $city->lat; $buffer = [];
$place->long = $city->lng;
$place->save();
});
$bar->advance();
} }
}
$this->insertBuffer($buffer, $count);
$bar->finish(); $bar->finish();
$this->info('Successfully imported ' . $count . ' entries.');
return; return;
} }
private function insertBuffer($buffer, $count)
{
DB::table('places')->insert($buffer);
}
} }