(jenkins)
76 lines
2.4 KiB
JavaScript
76 lines
2.4 KiB
JavaScript
const { Client } = require('pg');
|
|
const axios = require('axios');
|
|
require('dotenv').config();
|
|
|
|
const DATA_URL = 'https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_populated_places_simple.geojson';
|
|
|
|
async function importCities() {
|
|
const client = new Client({
|
|
connectionString: process.env.DATABASE_URL || 'postgresql://line_of_sight:line_of_sight_pass@postgres:5432/line_of_sight'
|
|
});
|
|
|
|
try {
|
|
console.log('Connecting to database...');
|
|
await client.connect();
|
|
|
|
console.log('Downloading Natural Earth data (GeoJSON)...');
|
|
const response = await axios.get(DATA_URL);
|
|
const data = response.data;
|
|
|
|
console.log(`Downloaded ${data.features.length} features. Preparing database...`);
|
|
|
|
// Sample properties to verify structure
|
|
if (data.features.length > 0) {
|
|
console.log('Sample properties:', data.features[0].properties);
|
|
}
|
|
|
|
// Ensure table exists and is clean
|
|
await client.query('TRUNCATE TABLE cities');
|
|
|
|
let count = 0;
|
|
const batchSize = 100;
|
|
|
|
for (let i = 0; i < data.features.length; i += batchSize) {
|
|
const batch = data.features.slice(i, i + batchSize);
|
|
|
|
const queryParts = [];
|
|
const values = [];
|
|
|
|
batch.forEach((feature, index) => {
|
|
const p = feature.properties;
|
|
const c = feature.geometry.coordinates;
|
|
|
|
// Map properties - Natural Earth simple GeoJSON usually has name, pop_max, adm0name
|
|
const name = p.name || p.NAME || 'Unknown';
|
|
const pop = p.pop_max || p.POP_MAX || 0;
|
|
const country = p.adm0name || p.ADM0NAME || 'Unknown';
|
|
const lon = c[0];
|
|
const lat = c[1];
|
|
|
|
const base = index * 5;
|
|
queryParts.push(`($${base + 1}, $${base + 2}, $${base + 3}, ST_SetSRID(ST_MakePoint($${base + 4}, $${base + 5}), 4326)::geography)`);
|
|
values.push(name, pop, country, lon, lat);
|
|
});
|
|
|
|
await client.query(
|
|
`INSERT INTO cities (name, population, country, geom) VALUES ${queryParts.join(',')}`,
|
|
values
|
|
);
|
|
|
|
count += batch.length;
|
|
if (count % 1000 === 0 || count === data.features.length) {
|
|
console.log(`Imported ${count}/${data.features.length} cities...`);
|
|
}
|
|
}
|
|
|
|
console.log('SUCCESS: Natural Earth data import complete.');
|
|
|
|
} catch (err) {
|
|
console.error('ERROR during import:', err);
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
importCities();
|