Implement real GIS data import and PostGIS spatial queries

2026-03-16 20:32:22 +00:00
parent e764efb189
commit ad07a70125
4 changed files with 312 additions and 56 deletions
@@ -1,66 +1,106 @@
 const express = require('express');
 const cors = require('cors');
+const { Pool } = require('pg');
 require('dotenv').config();

 const app = express();
 const PORT = process.env.PORT || 3001;

+// Database connection pool
+const pool = new Pool({
+  connectionString: process.env.DATABASE_URL || 'postgresql://line_of_sight:line_of_sight_pass@postgres:5432/line_of_sight'
+});
+
 app.use(cors());
 app.use(express.json());

-// Mock conurbation data for MVP
-const MOCK_CONURBATIONS = [
-  { id: 1, name: "London", population: 9000000, distance_km: 0, lat: 51.5074, lon: -0.1278 },
-  { id: 2, name: "Paris", population: 2161000, distance_km: 344, lat: 48.8566, lon: 2.3522 },
-  { id: 3, name: "Berlin", population: 3644000, distance_km: 878, lat: 52.5200, lon: 13.4050 },
-  { id: 4, name: "Warsaw", population: 1793000, distance_km: 1200, lat: 52.2297, lon: 21.0122 },
-  { id: 5, name: "Moscow", population: 12506000, distance_km: 2063, lat: 55.7558, lon: 37.6173 },
-  { id: 6, name: "Kazan", population: 1257000, distance_km: 2850, lat: 55.7897, lon: 49.1219 },
-  { id: 7, name: "Almaty", population: 2000000, distance_km: 3900, lat: 43.2220, lon: 76.8512 },
-  { id: 8, name: "Urumqi", population: 3500000, distance_km: 4500, lat: 43.8256, lon: 87.6168 },
-  { id: 9, name: "Lahore", population: 11126000, distance_km: 5400, lat: 31.5204, lon: 74.3587 },
-  { id: 10, name: "New Delhi", population: 29399000, distance_km: 5800, lat: 28.6139, lon: 77.2090 },
-  { id: 11, name: "Dhaka", population: 21006000, distance_km: 6200, lat: 23.8103, lon: 90.4125 },
-  { id: 12, name: "Chennai", population: 10971000, distance_km: 6500, lat: 13.0827, lon: 80.2707 },
-  { id: 13, name: "Bangkok", population: 10539000, distance_km: 7200, lat: 13.7563, lon: 100.5018 },
-  { id: 14, name: "Jakarta", population: 10562000, distance_km: 8100, lat: -6.2088, lon: 106.8456 },
-  { id: 15, name: "Singapore", population: 5686000, distance_km: 8300, lat: 1.3521, lon: 103.8198 },
-  { id: 16, name: "Manila", population: 17801000, distance_km: 8700, lat: 14.5995, lon: 120.9842 },
-  { id: 17, name: "Tokyo", population: 37400000, distance_km: 9500, lat: 35.6762, lon: 139.6503 },
-  { id: 18, name: "Seoul", population: 9720000, distance_km: 9200, lat: 37.5665, lon: 126.9780 },
-  { id: 19, name: "Beijing", population: 21540000, distance_km: 8900, lat: 39.9042, lon: 116.4074 },
-  { id: 20, name: "Shanghai", population: 27058000, distance_km: 9000, lat: 31.2304, lon: 121.4737 }
-];
+// Helper to calculate destination point given start, bearing, and distance (km)
+const calculateDestination = (lat, lon, bearing, distance) => {
+  const R = 6371;
+  const brng = (bearing * Math.PI) / 180;
+  const φ1 = (lat * Math.PI) / 180;
+  const λ1 = (lon * Math.PI) / 180;
+  const δ = distance / R;

-// Mock API endpoint - returns dummy conurbations based on input coordinates
-app.get('/api/line-of-sight', (req, res) => {
+  const φ2 = Math.asin(
+    Math.sin(φ1) * Math.cos(δ) +
+      Math.cos(φ1) * Math.sin(δ) * Math.cos(brng)
+  );
+  const λ2 =
+    λ1 +
+    Math.atan2(
+      Math.sin(brng) * Math.sin(δ) * Math.cos(φ1),
+      Math.cos(δ) - Math.sin(φ1) * Math.sin(φ2)
+    );
+
+  return {
+    lat: (φ2 * 180) / Math.PI,
+    lon: (((λ2 * 180) / Math.PI + 540) % 360) - 180
+  };
+};
+
+// Real API endpoint - uses PostGIS for spatial queries
+app.get('/api/line-of-sight', async (req, res) => {
  const { lat, lon, direction, tolerance } = req.query;
  
-  console.log(`Received request: lat=${lat}, lon=${lon}, direction=${direction}, tolerance=${tolerance}`);
-  
-  // Return mock data for MVP
-  res.json({
-    success: true,
-    data: {
-      start_point: { lat: parseFloat(lat) || 51.5074, lon: parseFloat(lon) || -0.1278 },
-      direction: parseInt(direction) || 45,
-      tolerance_km: parseInt(tolerance) || 50,
-      conurbations: MOCK_CONURBATIONS.slice(0, 20),
-      line_coordinates: [
-        { lat: 51.5074, lon: -0.1278 },
-        { lat: 48.8566, lon: 2.3522 },
-        { lat: 52.5200, lon: 13.4050 },
-        { lat: 55.7558, lon: 37.6173 },
-        { lat: 43.2220, lon: 76.8512 },
-        { lat: 28.6139, lon: 77.2090 },
-        { lat: 13.7563, lon: 100.5018 },
-        { lat: -6.2088, lon: 106.8456 },
-        { lat: 35.6762, lon: 139.6503 },
-        { lat: 51.5074, lon: -0.1278 } // Complete the circle
-      ]
-    },
-    message: "Mock data returned for MVP - Real geospatial calculations coming soon"
-  });
+  const startLat = parseFloat(lat) || 51.5074;
+  const startLon = parseFloat(lon) || -0.1278;
+  const bearing = parseInt(direction) || 0;
+  const toleranceKm = parseInt(tolerance) || 50;
+
+  console.log(`Processing real request: lat=${startLat}, lon=${startLon}, bearing=${bearing}, tolerance=${toleranceKm}`);
+
+  try {
+    // Generate path points for visualization and spatial query
+    const pathPoints = [];
+    const totalDistance = 10000;
+    const steps = 20;
+    
+    for (let i = 0; i <= steps; i++) {
+      const dist = (totalDistance * i) / steps;
+      pathPoints.push(calculateDestination(startLat, startLon, bearing, dist));
+    }
+
+    const lineWKT = `LINESTRING(${pathPoints.map(p => `${p.lon} ${p.lat}`).join(',')})`;
+    
+    const query = `
+      WITH path AS (
+        SELECT ST_GeogFromText($1) as route
+      )
+      SELECT 
+        id, 
+        name, 
+        population, 
+        country,
+        ST_Y(geom::geometry) as lat, 
+        ST_X(geom::geometry) as lon,
+        ST_Distance(geom, (SELECT route FROM path)) / 1000 as distance_to_line_km,
+        ST_LineLocatePoint((SELECT route FROM path)::geometry, geom::geometry) as pos_on_line
+      FROM cities
+      WHERE ST_DWithin(geom, (SELECT route FROM path), $2 * 1000)
+      ORDER BY pos_on_line ASC
+      LIMIT 20;
+    `;
+
+    const result = await pool.query(query, [lineWKT, toleranceKm]);
+
+    res.json({
+      success: true,
+      data: {
+        start_point: { lat: startLat, lon: startLon },
+        direction: bearing,
+        tolerance_km: toleranceKm,
+        conurbations: result.rows.map(row => ({
+          ...row,
+          distance_km: Math.round(row.distance_to_line_km)
+        })),
+        line_coordinates: pathPoints
+      }
+    });
+  } catch (err) {
+    console.error('Database query error:', err);
+    res.status(500).json({ success: false, error: 'Database query failed' });
+  }
 });

 // Health check endpoint
@@ -8,6 +8,7 @@
      "name": "line-of-sight-backend",
      "version": "1.0.0",
      "dependencies": {
+        "axios": "^1.13.6",
        "cors": "^2.8.6",
        "dotenv": "^17.3.1",
        "express": "^5.2.1",
@@ -1426,6 +1427,23 @@
        "sprintf-js": "~1.0.2"
      }
    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "license": "MIT"
+    },
+    "node_modules/axios": {
+      "version": "1.13.6",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.6.tgz",
+      "integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==",
+      "license": "MIT",
+      "dependencies": {
+        "follow-redirects": "^1.15.11",
+        "form-data": "^4.0.5",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
    "node_modules/babel-jest": {
      "version": "30.3.0",
      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.3.0.tgz",
@@ -1891,6 +1909,18 @@
      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
      "dev": true
    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "license": "MIT",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
    "node_modules/concat-map": {
      "version": "0.0.1",
      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -2008,6 +2038,15 @@
        "node": ">=0.10.0"
      }
    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
    "node_modules/depd": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -2128,6 +2167,21 @@
        "node": ">= 0.4"
      }
    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
    "node_modules/escalade": {
      "version": "3.2.0",
      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
@@ -2329,6 +2383,26 @@
        "node": ">=8"
      }
    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.11",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
+      "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "license": "MIT",
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/foreground-child": {
      "version": "3.3.1",
      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",
@@ -2345,6 +2419,43 @@
        "url": "https://github.com/sponsors/isaacs"
      }
    },
+    "node_modules/form-data": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
+      "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/form-data/node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/form-data/node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
    "node_modules/forwarded": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -2533,6 +2644,21 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "license": "MIT",
+      "dependencies": {
+        "has-symbols": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
    "node_modules/hasown": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@@ -4158,6 +4284,12 @@
        "node": ">= 0.10"
      }
    },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
+      "license": "MIT"
+    },
    "node_modules/pstree.remy": {
      "version": "1.1.8",
      "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
@@ -6,16 +6,18 @@
  "scripts": {
    "start": "node app/server.js",
    "dev": "nodemon app/server.js",
-    "test": "jest"
+    "test": "jest",
+    "seed-data": "node scripts/import_cities.js"
  },
  "dependencies": {
-    "express": "^5.2.1",
+    "axios": "^1.13.6",
    "cors": "^2.8.6",
-    "pg": "^8.20.0",
-    "dotenv": "^17.3.1"
+    "dotenv": "^17.3.1",
+    "express": "^5.2.1",
+    "pg": "^8.20.0"
  },
  "devDependencies": {
-    "nodemon": "^3.1.14",
-    "jest": "^30.3.0"
+    "jest": "^30.3.0",
+    "nodemon": "^3.1.14"
  }
 }
@@ -0,0 +1,82 @@
+const { Client } = require('pg');
+const axios = require('axios');
+require('dotenv').config();
+
+const DATA_URL = 'https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_populated_places_simple.geojson';
+
+async function importCities() {
+  const client = new Client({
+    connectionString: process.env.DATABASE_URL || 'postgresql://line_of_sight:line_of_sight_pass@localhost:5432/line_of_sight'
+  });
+
+  try {
+    console.log('Connecting to database...');
+    await client.connect();
+
+    console.log('Downloading Natural Earth data (GeoJSON)...');
+    const response = await axios.get(DATA_URL);
+    const data = response.data;
+
+    console.log(`Downloaded ${data.features.length} features. Preparing database...`);
+
+    // Ensure table exists and is clean
+    await client.query('TRUNCATE TABLE cities');
+
+    let count = 0;
+    const batchSize = 100;
+
+    for (let i = 0; i < data.features.length; i += batchSize) {
+      const batch = data.features.slice(i, i + batchSize);
+      
+      const values = [];
+      const queryParts = [];
+      
+      batch.forEach((feature, index) => {
+        const props = feature.properties;
+        const coords = feature.geometry.coordinates; // [lon, lat]
+        
+        const name = props.NAME || 'Unknown';
+        const population = props.POP_MAX || 0;
+        const country = props.ADM0NAME || 'Unknown';
+        const lon = coords[0];
+        const lat = coords[1];
+
+        const baseIndex = index * 4;
+        queryParts.push(`($${baseIndex + 1}, $${baseIndex + 2}, $${baseIndex + 3}, ST_SetSRID(ST_MakePoint($${baseIndex + 4}, $${baseIndex + 1}), 4326)::geography)`);
+        values.push(lat, name, population, country, lon); // Note: ST_MakePoint takes lon, lat
+      });
+
+      // Simple positional mapping for query (lat, name, pop, country, lon)
+      // Actually let's refine the query to be clearer
+      const refinedQueryParts = [];
+      const refinedValues = [];
+      
+      batch.forEach((feature, index) => {
+        const p = feature.properties;
+        const c = feature.geometry.coordinates;
+        const base = index * 5;
+        refinedQueryParts.push(`($${base + 1}, $${base + 2}, $${base + 3}, ST_SetSRID(ST_MakePoint($${base + 4}, $${base + 5}), 4326)::geography)`);
+        refinedValues.push(p.NAME || 'Unknown', p.POP_MAX || 0, p.ADM0NAME || 'Unknown', c[0], c[1]);
+      });
+
+      await client.query(
+        `INSERT INTO cities (name, population, country, geom) VALUES ${refinedQueryParts.join(',')}`,
+        refinedValues
+      );
+      
+      count += batch.length;
+      if (count % 1000 === 0 || count === data.features.length) {
+        console.log(`Imported ${count}/${data.features.length} cities...`);
+      }
+    }
+
+    console.log('SUCCESS: Natural Earth data import complete.');
+
+  } catch (err) {
+    console.error('ERROR during import:', err);
+  } finally {
+    await client.end();
+  }
+}
+
+importCities();