Implement 3D globe fly-over animation using MapLibre FreeCamera and Turf.js

backend/scripts/import_cities.js

@@ -1,75 +1,95 @@
-const { Client } = require('pg');
-const axios = require('axios');
+const { Pool } = require('pg');
+const { execSync } = require('child_process');
+const fs = require('fs');
+const readline = require('readline');
 require('dotenv').config();
 
-const DATA_URL = 'https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_populated_places_simple.geojson';
+const DATA_URL = 'https://download.geonames.org/export/dump/cities5000.zip';
+const ZIP_FILE = '/tmp/cities.zip';
+const TXT_FILE = '/tmp/cities5000.txt';
 
-async function importCities() {
-  const client = new Client({
+async function importGeoNames() {
+  const pool = new Pool({
     connectionString: process.env.DATABASE_URL || 'postgresql://line_of_sight:line_of_sight_pass@postgres:5432/line_of_sight'
   });
 
   try {
+    console.log('Downloading GeoNames cities5000 (Pop > 5000)...');
+    execSync(`wget -q ${DATA_URL} -O ${ZIP_FILE}`);
+    
+    console.log('Extracting data...');
+    execSync(`unzip -o ${ZIP_FILE} -d /tmp`);
+
     console.log('Connecting to database...');
-    await client.connect();
+    const client = await pool.connect();
 
-    console.log('Downloading Natural Earth data (GeoJSON)...');
-    const response = await axios.get(DATA_URL);
-    const data = response.data;
-
-    console.log(`Downloaded ${data.features.length} features. Preparing database...`);
-
-    // Sample properties to verify structure
-    if (data.features.length > 0) {
-      console.log('Sample properties:', data.features[0].properties);
-    }
-
-    // Ensure table exists and is clean
+    // Ensure table is clean
     await client.query('TRUNCATE TABLE cities');
 
+    console.log('Starting stream import...');
+    
+    const fileStream = fs.createReadStream(TXT_FILE);
+    const rl = readline.createInterface({
+      input: fileStream,
+      crlfDelay: Infinity
+    });
+
+    let batch = [];
+    const batchSize = 500;
     let count = 0;
-    const batchSize = 100;
 
-    for (let i = 0; i < data.features.length; i += batchSize) {
-      const batch = data.features.slice(i, i + batchSize);
-      
-      const queryParts = [];
-      const values = [];
-      
-      batch.forEach((feature, index) => {
-        const p = feature.properties;
-        const c = feature.geometry.coordinates;
-        
-        // Map properties - Natural Earth simple GeoJSON usually has name, pop_max, adm0name
-        const name = p.name || p.NAME || 'Unknown';
-        const pop = p.pop_max || p.POP_MAX || 0;
-        const country = p.adm0name || p.ADM0NAME || 'Unknown';
-        const lon = c[0];
-        const lat = c[1];
+    for await (const line of rl) {
+      const parts = line.split('\t');
+      if (parts.length < 15) continue;
 
-        const base = index * 5;
-        queryParts.push(`($${base + 1}, $${base + 2}, $${base + 3}, ST_SetSRID(ST_MakePoint($${base + 4}, $${base + 5}), 4326)::geography)`);
-        values.push(name, pop, country, lon, lat);
-      });
+      const name = parts[1]; // name
+      const lat = parseFloat(parts[4]);
+      const lon = parseFloat(parts[5]);
+      const country = parts[8]; // country code
+      const population = parseInt(parts[14]) || 0;
 
-      await client.query(
-        `INSERT INTO cities (name, population, country, geom) VALUES ${queryParts.join(',')}`,
-        values
-      );
-      
-      count += batch.length;
-      if (count % 1000 === 0 || count === data.features.length) {
-        console.log(`Imported ${count}/${data.features.length} cities...`);
+      batch.push({ name, lat, lon, country, population });
+
+      if (batch.length >= batchSize) {
+        await insertBatch(client, batch);
+        count += batch.length;
+        if (count % 5000 === 0) console.log(`Imported ${count} cities...`);
+        batch = [];
       }
     }
 
-    console.log('SUCCESS: Natural Earth data import complete.');
+    if (batch.length > 0) {
+      await insertBatch(client, batch);
+      count += batch.length;
+    }
+
+    console.log(`SUCCESS: Imported ${count} cities and towns.`);
+    client.release();
 
   } catch (err) {
     console.error('ERROR during import:', err);
   } finally {
-    await client.end();
+    // Cleanup
+    if (fs.existsSync(ZIP_FILE)) fs.unlinkSync(ZIP_FILE);
+    if (fs.existsSync(TXT_FILE)) fs.unlinkSync(TXT_FILE);
+    await pool.end();
   }
 }
 
-importCities();
+async function insertBatch(client, batch) {
+  const queryParts = [];
+  const values = [];
+  
+  batch.forEach((city, index) => {
+    const base = index * 5;
+    queryParts.push(`($${base + 1}, $${base + 2}, $${base + 3}, ST_SetSRID(ST_MakePoint($${base + 4}, $${base + 5}), 4326)::geography)`);
+    values.push(city.name, city.population, city.country, city.lon, city.lat);
+  });
+
+  await client.query(
+    `INSERT INTO cities (name, population, country, geom) VALUES ${queryParts.join(',')}`,
+    values
+  );
+}
+
+importGeoNames();