Skip to content

GitLab

Commit 3b9be901 authored by alain's avatar alain 🐙
Browse files

api endoint based on env

parent ca6a8897
Hide whitespace changes
Inline Side-by-side
Showing with 200 additions and 314 deletions
frontend/config/data.js
View file @ 3b9be901
import luchtmeetnet from "./sources/luchtmeetnet"
import teom from "./sources/teom"
//import holu from "./sources/holu"
import holu from "./sources/holu-cal"
//import holu from "./sources/holu-rivm"
import holu from "./sources/holu"
//import sentinel from "./sources/sentinel"
const colors = {
......
frontend/config/sources/holu-cal.js deleted 100644 → 0
View file @ ca6a8897
import { addMissingDataPoints } from "../../data-on-a-map-app/src/util/data"
export default {
name: "holu",
sides: 24,
angle: 0,
factor: 0.5,
daysToFetch: 90,
dataStart: "2019-06-25T00:00:00.000Z",
dataStreams: {
pm25: {
raw: {
minmax: { key: "pm25_raw_minmax", type: "area", active: false, color: false },
mean: { key: "pm25_raw_mean", type: "line", active: false, color: false }
},
cal: {
minmax: { key: "pm25_cal_minmax", type: "area", active: true, color: true },
mean: { key: "pm25_cal_mean", type: "line", active: true, color: true }
}
},
pm10: {
raw: {
minmax: { key: "pm10_raw_minmax", type: "area", active: false, color: false },
mean: { key: "pm10_raw_mean", type: "line", active: false, color: false }
},
cal: {
minmax: { key: "pm10_cal_minmax", type: "area", active: true, color: true },
mean: { key: "pm10_cal_mean", type: "line", active: true, color: true }
}
},
no2: {
single: {
mean: { key: "no2_mean", type: "line", active: true, color: true }
}
}
},
all: async function(parameter) {
try {
const allResponse = await fetch(`https://data.waag.org/api/getStations`)
const all = await allResponse.json()
const latestRawResponse = await fetch(`https://data.waag.org/api/getAllSensors`)
let latestRaw = await latestRawResponse.json()
const latestResponse = await fetch(`https://data.waag.org/api/holu/holukit/calibrated/recent`)
let latest = await latestResponse.json()
let offlineSensors = []
let unregisteredSensors = []
//let places = {}
const filterExtended = parameter === "no2" ? true : false
const data = all.filter(sensor => {
return filterExtended ? sensor.extended : true
}).map(sensor => {
const kitType = sensor.id > 1000000 ? "MUV-kit" : "HOLU-kit"
const idPrefix = sensor.id > 1000000 ? "HLL_MUV_device_" : ""
const sensorData = latest.find(o => o.id.toString().replace(idPrefix, "") === sensor.id.toString())
const sensorDataReturn = {
id: idPrefix + sensor.id,
name: `${kitType} ${sensor.id.toString()}: ${sensor.tag} - ${sensor.name}`,
source: "holu",
type: sensor.extended ? "extended" : "basic",
coordinates: [sensor.longitude, sensor.latitude],
mean: null,
timestamp: null,
dataAge: null,
status: null
}
if(sensorData && sensorData[`${parameter}_mean`] !== 0) {
const dataAgeInHours = ((new Date() - new Date(sensorData.time)) / 3600000) - 1
if(dataAgeInHours >= 48 || dataAgeInHours < 0) {
sensorDataReturn.status = 0
offlineSensors.push(sensor.id)
} else if(dataAgeInHours > 8) {
sensorDataReturn.status = 1
} else {
sensorDataReturn.mean = sensorData[`${parameter}_mean`]
}
sensorDataReturn.timestamp = sensorData.time
sensorDataReturn.dataAge = dataAgeInHours
} else {
const sensorDataRaw = latestRaw.find(o => +o.id === +sensor.id)
if(sensorDataRaw) {
sensorDataReturn.status = 2
} else {
sensorDataReturn.status = 0
}
offlineSensors.push(sensor.id)
}
return sensorDataReturn
})
latest.forEach(sensor => {
if(!all.find(o => +o.id === +sensor.id)) {
unregisteredSensors.push(sensor.id)
}
})
console.log(`Offline Sensors (${offlineSensors.length}):`, offlineSensors.sort((a, b) => a - b).join(', '))
console.log(`Unregistered Sensors (${unregisteredSensors.length}):`, unregisteredSensors.sort((a, b) => a - b).join(', '))
// Object.keys(places).forEach(function(place) {
// places[place].amountOfSensors = places[place].allAverages.length
// places[place].averageOfAverages = places[place].allAverages.reduce((a, b) => a + b, 0) / places[place].allAverages.length
// places[place].averageOfPeaks = places[place].allPeaks.reduce((a, b) => a + b, 0) / places[place].allPeaks.length
// places[place].highestAverage = Math.max(...places[place].allAverages)
// places[place].highestPeak = Math.max(...places[place].allPeaks)
// })
// console.log(places)
return data
} catch (error) {
console.log(error)
}
},
stationMeta: function(station, parameter) {
let description
if(station.name.includes('HOLU')) {
description = (station.type === "extended" ? (
'<span>Dit is een extended HOLU-meetstation. Dit prototype meet PM2.5 en PM10 en ook NO2 en O3. Als onderdeel van de pilot wordt er onderzoek gedaan naar de kwaliteit van de data afkomstig van deze meetstations.<br /><a class="more" href="https://hollandseluchten.waag.org/holu-sensorkit/">Lees meer over de HOLU kit</a></span>'
) : (
'<span>Dit is een basic HOLU-meetstation. Dit prototype meet PM2.5 en PM10. Als onderdeel van de pilot wordt er onderzoek gedaan naar de kwaliteit van de data afkomstig van deze meetstations.<br /><a class="more" href="https://hollandseluchten.waag.org/holu-sensorkit/">Lees meer over de HOLU kit</a></span>'
)
)
} else {
description = '<span>Dit is een MUV monitoring station.</span>'
}
return {
name: station.name,
description: description
}
},
stationData: async function(station, parameter, start, end, granularity) {
try {
var prefix
if (parameter === "pm25") prefix = parameter
if (parameter === "pm10") prefix = parameter
if (parameter === "no2") prefix = parameter
if (parameter === "temperature") prefix = "t_out"
if (parameter === "humidity") prefix = "h_out"
let data = []
if(parameter === "pm25" || parameter === "pm10") {
let rawURL = `https://data.waag.org/api/getHourlyValuesSensor?sensor_id=${station.id}&start=${start}&end=${end}`
if(station.id.includes('MUV')) {
rawURL = `https://data.waag.org/api/muv/ms/hourly?sensor_id=${station.id.replace('HLL_MUV_device_','')}&start=${start}&end=${end}`
}
const responseRaw = await fetch(rawURL)
const jsonRaw = await responseRaw.json()
let dataRaw = jsonRaw.map(m => {
return {
timestamp: new Date(m.time).getTime(),
mean: m[`${prefix}_mean`],
minmax: [m[`${prefix}_min`], m[`${prefix}_max`]]
}
})
const responseCal = await fetch(`https://data.waag.org/api/holu/holukit/calibrated/hourly?sensor_id=${station.id}&start=${start}&end=${end}`)
const jsonCal = await responseCal.json()
let dataCal = jsonCal.map(m => {
return {
timestamp: new Date(m.time).getTime(),
mean: m[`${prefix}_mean`],
fac: m[`${prefix}_fac`],
}
})
if(dataRaw[0] && dataCal[0]) {
const dataStart = (dataRaw[0].timestamp < dataCal[0].timestamp ? dataRaw[0].timestamp : dataCal[0].timestamp)
const dataEnd = new Date(end).getTime()
dataRaw = addMissingDataPoints(dataRaw, dataStart, dataEnd, granularity)
dataCal = addMissingDataPoints(dataCal, dataStart, dataEnd, granularity)
}
for (let i = 0; i < dataRaw.length; i++) {
const r = dataRaw[i]
const c = dataCal[i]
const o = {
timestamp: r.timestamp
}
o[`${prefix}_raw_mean`] = r.mean
o[`${prefix}_raw_minmax`] = r.minmax
o[`${prefix}_cal_mean`] = c.mean
if(r.mean && c.mean) {
o[`${prefix}_cal_minmax`] = [r.minmax[0] * c.fac, r.minmax[1] * c.fac]
}
data.push(o)
}
} else {
const response = await fetch(`https://data.waag.org/api/holu/holukit/calibrated/hourly?sensor_id=${station.id}&start=${start}&end=${end}`)
const json = await response.json()
data = json.map(m => {
m.timestamp = new Date(m.time).getTime()
delete m.time
if(m[`${prefix}_mean`] === -999) m[`${prefix}_mean`] = null
return m
})
if(data[0]) {
const dataStart = data[0].timestamp
const dataEnd = new Date(end).getTime()
data = addMissingDataPoints(data, dataStart, dataEnd, granularity)
}
}
return { status: "success", data }
} catch (error) {
console.log(error)
return { status: "error" }
}
},
download: function(id, parameter, start, end) {
return `https://data.waag.org/api/holu/holukit/calibrated/hourly?sensor_id=${id}&start=${start}&end=${end}&format=csv`
}
}
\ No newline at end of file
frontend/config/sources/holu.js
View file @ 3b9be901
import { addMissingDataPoints } from "../../data-on-a-map-app/src/util/data"
const endPointDataPlatform = process.env.NODE_ENV === 'development' ? "http://localhost:1881" : "https://data.waag.org"
const endPointDataPlatformMuv = process.env.NODE_ENV === 'development' ? "http://localhost:1886" : "https://data.waag.org"
export default {
name: "holu",
sides: 24,
angle: 0,
factor: 0.5,
daysToFetch: 90,
dataStart: "2019-06-25T00:00:00.000Z",
dataStreams: {
pm25: {
raw: {
minmax: { key: "pm25_raw_minmax", type: "area", active: false, color: false },
mean: { key: "pm25_raw_mean", type: "line", active: false, color: false }
},
cal: {
minmax: { key: "pm25_cal_minmax", type: "area", active: true, color: true },
mean: { key: "pm25_cal_mean", type: "line", active: true, color: true }
}
},
pm10: {
raw: {
minmax: { key: "pm10_raw_minmax", type: "area", active: false, color: false },
mean: { key: "pm10_raw_mean", type: "line", active: false, color: false }
},
cal: {
minmax: { key: "pm10_cal_minmax", type: "area", active: true, color: true },
mean: { key: "pm10_cal_mean", type: "line", active: true, color: true }
}
},
no2: {
single: {
mean: { key: "no2_mean", type: "line", active: true, color: true }
}
}
},
all: async function(parameter) {
try {
const allResponse = await fetch(`https://data.waag.org/api/getStations`)
const allResponse = await fetch(`${endPointDataPlatform}/api/getStations`)
const all = await allResponse.json()
const latestResponse = await fetch(`https://data.waag.org/api/getAllSensors`)
const latestRawResponse = await fetch(`${endPointDataPlatform}/api/getAllSensors`)
let latestRaw = await latestRawResponse.json()
const latestResponse = await fetch(`${endPointDataPlatform}/api/holu/holukit/calibrated/recent`)
let latest = await latestResponse.json()
let offlineSensors = []
let unregisteredSensors = []
let places = {}
const data = all.map(sensor => {
const sensorData = latest.find(o => +o.id === +sensor.id)
//let places = {}
if(sensorData) {
if(!places[sensor.tag]) places[sensor.tag] = {
allAverages: [],
allPeaks: []
}
places[sensor.tag].allAverages.push(sensorData[`${parameter}_mean`])
places[sensor.tag].allPeaks.push(sensorData[`${parameter}_max`])
const filterExtended = parameter === "no2" ? true : false
return {
id: sensor.id.toString(),
name: "HOLU-sensor " + sensor.id.toString() + ": " + sensor.name,
type: "HOLU-sensor",
source: "holu",
coordinates: [sensor.longitude, sensor.latitude],
mean: sensorData[`${parameter}_mean`],
max: sensorData[`${parameter}_max`]
const data = all.filter(sensor => {
return filterExtended ? sensor.extended : true
}).map(sensor => {
const kitType = sensor.id > 1000000 ? "MUV-kit" : "HOLU-kit"
const idPrefix = sensor.id > 1000000 ? "HLL_MUV_device_" : ""
const sensorData = latest.find(o => o.id.toString().replace(idPrefix, "") === sensor.id.toString())
const sensorDataReturn = {
id: idPrefix + sensor.id,
name: `${kitType} ${sensor.id.toString()}: ${sensor.tag} - ${sensor.name}`,
source: "holu",
type: sensor.extended ? "extended" : "basic",
coordinates: [sensor.longitude, sensor.latitude],
mean: null,
timestamp: null,
dataAge: null,
status: null
}
if(sensorData && sensorData[`${parameter}_mean`] !== 0) {
const dataAgeInHours = ((new Date() - new Date(sensorData.time)) / 3600000) - 1
if(dataAgeInHours >= 48 || dataAgeInHours < 0) {
sensorDataReturn.status = 0
offlineSensors.push(sensor.id)
} else if(dataAgeInHours > 8) {
sensorDataReturn.status = 1
} else {
sensorDataReturn.mean = sensorData[`${parameter}_mean`]
}
sensorDataReturn.timestamp = sensorData.time
sensorDataReturn.dataAge = dataAgeInHours
} else {
offlineSensors.push(sensor.id)
return {
id: sensor.id.toString(),
name: "HOLU-sensor " + sensor.id.toString() + ": " + sensor.name + " (offline)",
type: "HOLU-sensor",
source: "holu",
coordinates: [sensor.longitude, sensor.latitude],
mean: null,
max: null
const sensorDataRaw = latestRaw.find(o => +o.id === +sensor.id)
if(sensorDataRaw) {
sensorDataReturn.status = 2
} else {
sensorDataReturn.status = 0
}
offlineSensors.push(sensor.id)
}
return sensorDataReturn
})
latest.forEach(sensor => {
......@@ -64,14 +116,14 @@ export default {
console.log(`Offline Sensors (${offlineSensors.length}):`, offlineSensors.sort((a, b) => a - b).join(', '))
console.log(`Unregistered Sensors (${unregisteredSensors.length}):`, unregisteredSensors.sort((a, b) => a - b).join(', '))
Object.keys(places).forEach(function(place) {
places[place].amountOfSensors = places[place].allAverages.length
places[place].averageOfAverages = places[place].allAverages.reduce((a, b) => a + b, 0) / places[place].allAverages.length
places[place].averageOfPeaks = places[place].allPeaks.reduce((a, b) => a + b, 0) / places[place].allPeaks.length
places[place].highestAverage = Math.max(...places[place].allAverages)
places[place].highestPeak = Math.max(...places[place].allPeaks)
})
console.log(places)
// Object.keys(places).forEach(function(place) {
// places[place].amountOfSensors = places[place].allAverages.length
// places[place].averageOfAverages = places[place].allAverages.reduce((a, b) => a + b, 0) / places[place].allAverages.length
// places[place].averageOfPeaks = places[place].allPeaks.reduce((a, b) => a + b, 0) / places[place].allPeaks.length
// places[place].highestAverage = Math.max(...places[place].allAverages)
// places[place].highestPeak = Math.max(...places[place].allPeaks)
// })
// console.log(places)
return data
......@@ -81,44 +133,119 @@ export default {
},
stationMeta: function(station, parameter) {
const stationMeta = {
name: station.name,
description: (station.type === "extended" ? (
let description
if(station.name.includes('HOLU')) {
description = (station.type === "extended" ? (
'<span>Dit is een extended HOLU-meetstation. Dit prototype meet PM2.5 en PM10 en ook NO2 en O3. Als onderdeel van de pilot wordt er onderzoek gedaan naar de kwaliteit van de data afkomstig van deze meetstations.<br /><a class="more" href="https://hollandseluchten.waag.org/holu-sensorkit/">Lees meer over de HOLU kit</a></span>'
) : (
'<span>Dit is een basic HOLU-meetstation. Dit prototype meet PM2.5 en PM10. Als onderdeel van de pilot wordt er onderzoek gedaan naar de kwaliteit van de data afkomstig van deze meetstations.<br /><a class="more" href="https://hollandseluchten.waag.org/holu-sensorkit/">Lees meer over de HOLU kit</a></span>'
)
)
} else {
description = '<span>Dit is een MUV monitoring station.</span>'
}
return stationMeta
return {
name: station.name,
description: description
}
},
stationData: async function(station, parameter, start, end, granularity) {
try {
const response = await fetch(`https://data.waag.org/api/getHourlyValuesSensor?sensor_id=${station.id}&start=${start}&end=${end}`)
const responseData = await response.json()
var prefix
if (parameter === "pm25") prefix = parameter
if (parameter === "pm10") prefix = parameter
if (parameter === "no2") prefix = parameter
if (parameter === "temperature") prefix = "t_out"
if (parameter === "humidity") prefix = "h_out"
const conformedData = responseData.map(m => {
return {
timestamp: new Date(m.time).getTime(),
value: m[`${prefix}_mean`],
minmax: [m[`${prefix}_min`], m[`${prefix}_max`]]
let data = []
if(parameter === "pm25" || parameter === "pm10") {
let rawURL = `${endPointDataPlatform}/api/getHourlyValuesSensor?sensor_id=${station.id}&start=${start}&end=${end}`
if(station.id.includes('MUV')) {
rawURL = `${endPointDataPlatformMuv}/api/muv/ms/hourly?sensor_id=${station.id.replace('HLL_MUV_device_','')}&start=${start}&end=${end}`
}
})
const data = addMissingDataPoints(conformedData, granularity)
return data
const responseRaw = await fetch(rawURL)
const jsonRaw = await responseRaw.json()
let dataRaw = jsonRaw.map(m => {
return {
timestamp: new Date(m.time).getTime(),
mean: m[`${prefix}_mean`],
minmax: [m[`${prefix}_min`], m[`${prefix}_max`]]
}
})
const responseCal = await fetch(`${endPointDataPlatform}/api/holu/holukit/calibrated/hourly?sensor_id=${station.id}&start=${start}&end=${end}`)
const jsonCal = await responseCal.json()
let dataCal = jsonCal.map(m => {
return {
timestamp: new Date(m.time).getTime(),
mean: m[`${prefix}_mean`],
fac: m[`${prefix}_fac`],
}
})
if(dataRaw[0] && dataCal[0]) {
const dataStart = (dataRaw[0].timestamp < dataCal[0].timestamp ? dataRaw[0].timestamp : dataCal[0].timestamp)
const dataEnd = new Date(end).getTime()
dataRaw = addMissingDataPoints(dataRaw, dataStart, dataEnd, granularity)
dataCal = addMissingDataPoints(dataCal, dataStart, dataEnd, granularity)
}
for (let i = 0; i < dataRaw.length; i++) {
const r = dataRaw[i]
const c = dataCal[i]
const o = {
timestamp: r.timestamp
}
o[`${prefix}_raw_mean`] = r.mean
o[`${prefix}_raw_minmax`] = r.minmax
o[`${prefix}_cal_mean`] = c.mean
if(r.mean && c.mean) {
o[`${prefix}_cal_minmax`] = [r.minmax[0] * c.fac, r.minmax[1] * c.fac]
}
data.push(o)
}
} else {
const response = await fetch(`${endPointDataPlatform}/api/holu/holukit/calibrated/hourly?sensor_id=${station.id}&start=${start}&end=${end}`)
const json = await response.json()
data = json.map(m => {
m.timestamp = new Date(m.time).getTime()
delete m.time
if(m[`${prefix}_mean`] === -999) m[`${prefix}_mean`] = null
return m
})
if(data[0]) {
const dataStart = data[0].timestamp
const dataEnd = new Date(end).getTime()
data = addMissingDataPoints(data, dataStart, dataEnd, granularity)
}
}
return { status: "success", data }
} catch (error) {
console.log(error)
return "error"
return { status: "error" }
}
},
download: function(id, parameter, start, end) {
return `${endPointDataPlatform}/api/holu/holukit/calibrated/hourly?sensor_id=${id}&start=${start}&end=${end}&format=csv`
}
}
\ No newline at end of file
frontend/config/sources/luchtmeetnet.js
View file @ 3b9be901
import { getLastHourISO, getNowISO, subtractOneHourISO } from "../../data-on-a-map-app/src/util/time"
import { groupArrayOfObjectsBy, renameObjectKey, addMissingDataPoints } from "../../data-on-a-map-app/src/util/data"
const endPointDataPlatform = process.env.NODE_ENV === 'development' ? "http://localhost:1881" : "https://data.waag.org"
const officialStations = {
pm25: [
"NL49007",
......@@ -110,7 +114,7 @@ export default {
const lastHourISO = getLastHourISO()
const nowHourISO = getNowISO()
const allResponse = await fetch(`https://data.waag.org/api/getOfficialMeasurement?formula=${parameter.toUpperCase()}&start=${lastHourISO}&end=${nowHourISO}&station_id=${officialStations[parameter].join("&station_id=")}`)
const allResponse = await fetch(`${endPointDataPlatform}/api/getOfficialMeasurement?formula=${parameter.toUpperCase()}&start=${lastHourISO}&end=${nowHourISO}&station_id=${officialStations[parameter].join("&station_id=")}`)
const all = await allResponse.json()
const dataGrouped = groupArrayOfObjectsBy(all, "station_number")
......@@ -150,7 +154,7 @@ export default {