(jenkins)
83 lines
2.9 KiB
JavaScript
83 lines
2.9 KiB
JavaScript
const { Client } = require('pg');
|
|
const axios = require('axios');
|
|
require('dotenv').config();
|
|
|
|
const DATA_URL = 'https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_populated_places_simple.geojson';
|
|
|
|
async function importCities() {
|
|
const client = new Client({
|
|
connectionString: process.env.DATABASE_URL || 'postgresql://line_of_sight:line_of_sight_pass@localhost:5432/line_of_sight'
|
|
});
|
|
|
|
try {
|
|
console.log('Connecting to database...');
|
|
await client.connect();
|
|
|
|
console.log('Downloading Natural Earth data (GeoJSON)...');
|
|
const response = await axios.get(DATA_URL);
|
|
const data = response.data;
|
|
|
|
console.log(`Downloaded ${data.features.length} features. Preparing database...`);
|
|
|
|
// Ensure table exists and is clean
|
|
await client.query('TRUNCATE TABLE cities');
|
|
|
|
let count = 0;
|
|
const batchSize = 100;
|
|
|
|
for (let i = 0; i < data.features.length; i += batchSize) {
|
|
const batch = data.features.slice(i, i + batchSize);
|
|
|
|
const values = [];
|
|
const queryParts = [];
|
|
|
|
batch.forEach((feature, index) => {
|
|
const props = feature.properties;
|
|
const coords = feature.geometry.coordinates; // [lon, lat]
|
|
|
|
const name = props.NAME || 'Unknown';
|
|
const population = props.POP_MAX || 0;
|
|
const country = props.ADM0NAME || 'Unknown';
|
|
const lon = coords[0];
|
|
const lat = coords[1];
|
|
|
|
const baseIndex = index * 4;
|
|
queryParts.push(`($${baseIndex + 1}, $${baseIndex + 2}, $${baseIndex + 3}, ST_SetSRID(ST_MakePoint($${baseIndex + 4}, $${baseIndex + 1}), 4326)::geography)`);
|
|
values.push(lat, name, population, country, lon); // Note: ST_MakePoint takes lon, lat
|
|
});
|
|
|
|
// Simple positional mapping for query (lat, name, pop, country, lon)
|
|
// Actually let's refine the query to be clearer
|
|
const refinedQueryParts = [];
|
|
const refinedValues = [];
|
|
|
|
batch.forEach((feature, index) => {
|
|
const p = feature.properties;
|
|
const c = feature.geometry.coordinates;
|
|
const base = index * 5;
|
|
refinedQueryParts.push(`($${base + 1}, $${base + 2}, $${base + 3}, ST_SetSRID(ST_MakePoint($${base + 4}, $${base + 5}), 4326)::geography)`);
|
|
refinedValues.push(p.NAME || 'Unknown', p.POP_MAX || 0, p.ADM0NAME || 'Unknown', c[0], c[1]);
|
|
});
|
|
|
|
await client.query(
|
|
`INSERT INTO cities (name, population, country, geom) VALUES ${refinedQueryParts.join(',')}`,
|
|
refinedValues
|
|
);
|
|
|
|
count += batch.length;
|
|
if (count % 1000 === 0 || count === data.features.length) {
|
|
console.log(`Imported ${count}/${data.features.length} cities...`);
|
|
}
|
|
}
|
|
|
|
console.log('SUCCESS: Natural Earth data import complete.');
|
|
|
|
} catch (err) {
|
|
console.error('ERROR during import:', err);
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
importCities();
|