Commit bb2d9409 authored by alain's avatar alain 🐙
Browse files
parents 3c471fef 6120a234
......@@ -2,10 +2,19 @@
gem install sinatra
gem install json
gem install rest-client
gem install csv
# uitvoeren
ruby server.rb
# hollandse-luchten-data-app backend
voorbeeld call:
http://localhost:4567/sensor/204/pm10?start=2019-08-25&end=2019-08-31
voorbeeld calls:
http://localhost:4567/sensor/204/pm10?start=2019-08-25&end=2019-08-31 # geeft alle waardes tussen start en end voor type pm10 van device 204
http://localhost:4567/sensor/204/pm10/csv?start=2019-08-25&end=2019-08-31 # geeft een csv bestand met alle waardes tussen start en end voor type pm10 van device 204
http://localhost:4567/sensor/pm25 # geeft de laatste waarde voor ieder device van type pm25
http://localhost:4567/sensor/all # geeft de laatste waarde voor ieder device
......@@ -4,8 +4,11 @@ require 'sinatra'
require 'rest-client'
require 'json'
require 'redis'
require 'csv'
$base_url = "https://api-samenmeten.rivm.nl/v1.0"
$redis = Redis.new({:host => 'localhost', :port => 6379, :db => 2})
get '/version' do
'v0.1'
end
......@@ -14,41 +17,122 @@ before do
content_type 'application/json'
end
# 1. alle observations per station / sensor (a'la getHourlyValuesSensor)
get '/sensor/:type' do
type = params[:type]
uri = "#{$base_url}/Things?$filter=properties/project eq 'Hollandse Luchten'&$expand=Datastreams/Observations($orderby=phenomenonTime desc;$select=phenomenonTime,result;$top=1)"
if type != "all"
uri = "#{$base_url}/Things?$filter=properties/project eq 'Hollandse Luchten'&$expand=Datastreams($filter=contains(name,%27#{type}%27)),Datastreams/Observations($orderby=phenomenonTime desc;$select=phenomenonTime,result;$top=1)"
puts uri
end
body = nil
if($redis.exists uri)
body = $redis.get uri
puts "cache hit"
else
pages = []
getPageRecursive(pages,uri)
things = pages.map{|page|page["value"]}.flatten
result = things.map{|thing|
ds_list = thing["Datastreams"]
x = nil
if(ds_list != nil)
x = { thing["name"] => ds_list.map{|ds|
obs_list = ds["Observations"]
obs = obs_list != nil ? obs_list.first : nil
obs != nil ? { ds["name"] => {:timestamp => obs["phenomenonTime"], :value => obs["result"]} } : nil
}}
end
x
}
body = result.to_json
$redis.set uri, body, {:ex => 3600} #one hour TODO: maybe the number of seconds until the next whole hour?
end
body
end
# json download
get '/sensor/:id/:type' do
#id = "001"
#type = "no2"
retrieveSensorData(params)
end
# csv download
get '/sensor/:id/:type/csv' do
json = retrieveSensorData(params)
data = JSON.parse(json)
content_type 'application/csv'
attachment "#{params[:id]}_#{params[:type]}_data.csv"
#reformat to tabular data
to_csv(data)
end
# generate csv data
def to_csv(data)
CSV.generate do |csv|
csv << [:instance,:date,:value]
data.each do |ds|
ds.each_pair do |key,instance|
instance.each do |row|
# puts "#{key}, #{row["timestamp"]} #{row["value"]}"
csv << [key,row["timestamp"],row["value"]]
end
end
end
end
end
#convenience method for json / csv enpoint reuse
def retrieveSensorData(params)
id = params[:id]
type = params[:type]
#TODO: set default start and end?
#date_start = "2019-08-25"
#date_end = "2019-08-30"
date_start = params[:start]
date_end = params[:end]
uri = "#{$base_url}/Things?$filter=name%20eq%20%27HLL_hl_device_#{id}%27&$expand=Datastreams($filter=contains(name,%27#{type}%27))"
response = RestClient.get uri
#parse the list of data stream id's for this id, type combination
ds_list = JSON.parse(response)["value"].first["Datastreams"]
ds_ids = ds_list.map{|ds|ds["@iot.id"]}
ds_names = ds_list.map{|ds|ds["name"]}
#for each datastream retrieve the pages recursively
all_pages = ds_ids.map.with_index{|id,index|
uri = "#{$base_url}/Datastreams(#{id})/Observations?$orderby=phenomenonTime%20asc&$filter=date(phenomenonTime)%20gt%20date(%27#{date_start}%27)%20and%20date(phenomenonTime)%20lt%20date(%27#{date_end}%27)&$select=phenomenonTime,result"
pages = []
getPageRecursive(pages,uri)
all_values = []
pages.each do|page|
page_values = page["value"].map{|val| {:timestamp => val["phenomenonTime"], :value => val["result"] } }
all_values << page_values
end
{ds_names[index] => all_values.flatten}
}
all_pages.to_json #todo store this in redis cache
body = nil
if($redis.exists uri)
body = $redis.get uri
puts "cache hit"
else
response = RestClient.get uri
#parse the list of data stream id's for this id, type combination
ds_list = JSON.parse(response)["value"].first["Datastreams"]
ds_ids = ds_list.map{|ds|ds["@iot.id"]}
ds_names = ds_list.map{|ds|ds["name"]}
#for each datastream retrieve the pages recursively
all_pages = ds_ids.map.with_index{|id,index|
page_uri = "#{$base_url}/Datastreams(#{id})/Observations?$orderby=phenomenonTime%20asc&$filter=date(phenomenonTime)%20gt%20date(%27#{date_start}%27)%20and%20date(phenomenonTime)%20lt%20date(%27#{date_end}%27)&$select=phenomenonTime,result"
pages = []
getPageRecursive(pages,page_uri)
all_values = []
pages.each do|page|
page_values = page["value"].map{|val| {:timestamp => val["phenomenonTime"], :value => val["result"] } }
all_values << page_values
end
{ds_names[index] => all_values.flatten}
}
body = all_pages.to_json
$redis.set uri, body, {:ex => 3600} #TODO: or maybe the number of seconds until the next whole hour?
end
body
end
def getPageRecursive(pages,uri)
response = RestClient.get uri
page = JSON.parse(response)
......@@ -56,17 +140,3 @@ def getPageRecursive(pages,uri)
pages << page
getPageRecursive(pages,nextLink +"&$select=phenomenonTime,result") if (nextLink != nil and not nextLink.empty?)
end
# 1. alle observations per station / sensor (a'la getHourlyValuesSensor)
#stap 1. alle datastreams voor een sensor id bv 220, en een parameter bv pm10
# https://api-samenmeten.rivm.nl/v1.0/Things?$filter=name%20eq%20%27HLL_hl_device_220%27&$expand=Datastreams($filter=contains(name,%27pm10%27))
#stap 2. voor iedere datastream, alle observations binnen een date range ophalen, met een default date range tussen nu en 3 maanden geleden.
# https://api-samenmeten.rivm.nl/v1.0/Datastreams(103433)/Observations?$orderby=phenomenonTime%20asc&$filter=date(phenomenonTime)%20gt%20date(%272019-08-25%27)%20and%20date(phenomenonTime)%20lt%20date(%272019-08-27%27)
# dit is gepagede data met 20 waardes per page.
# https://api-samenmeten.rivm.nl/v1.0/Datastreams(102452)/Observations?$orderby=phenomenonTime%20asc&$filter=date(phenomenonTime)%20gt%20date(%272019-08-25%27)%20and%20date(phenomenonTime)%20lt%20date(%272019-08-28%27)&$count=true
# 2. geef me alle sensoren met het laatste uurgemiddelde (a'la getAllSensors)
#
# https://api-samenmeten.rivm.nl/v1.0/Things?$filter=properties/project eq 'Hollandse Luchten'&$expand=Datastreams/Observations($orderby=phenomenonTime desc;$select=phenomenonTime,result;$top=1)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment