Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

T689 and T690 #22

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

T689 and T690 #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ea93a4
OGR Source Listing: Only add the attributes to listing that are defin…
SoerenHoffstedt Dec 13, 2018
36a7115
GDAL Datasets: Allow fixed filename, even when time_interval is provi…
SoerenHoffstedt Dec 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 57 additions & 50 deletions src/util/gdal_timesnap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,60 +146,67 @@ GDALTimesnap::GDALDataLoadingInfo GDALTimesnap::getDataLoadingInfo(Json::Value d
throw NoRasterForGivenTimeException("Requested time is not in range of dataset");

if(datasetJson.isMember("time_interval") || channelJson.isMember("time_interval")) {
Json::Value timeInterval = channelJson.get("time_interval", datasetJson.get("time_interval", Json::Value(Json::objectValue)));
TimeUnit intervalUnit = GDALTimesnap::createTimeUnit(timeInterval.get("unit", "Month").asString());
int intervalValue = timeInterval.get("value", 1).asInt();


// doesn't work on older boost versions because seconds precision is limit to 32bit
// ptime start = from_time_t(static_cast<time_t>(time_start_mapping));
// ptime wanted = from_time_t(static_cast<time_t>(wantedTimeUnix));

ptime start = ptime(boost::gregorian::date(1970,1,1)) + milliseconds(static_cast<long>(time_start_mapping * 1000));
ptime wanted = ptime(boost::gregorian::date(1970,1,1)) + milliseconds(static_cast<long>(wantedTimeUnix * 1000));

//snap the time to the given interval
ptime snappedTimeStart = GDALTimesnap::snapToInterval(intervalUnit, intervalValue, start, wanted);

// snap end to the next interval
ptime snappedTimeEnd(snappedTimeStart);
switch (intervalUnit) {
case TimeUnit::Year:
snappedTimeEnd += boost::gregorian::years(intervalValue);
break;
case TimeUnit::Month:
snappedTimeEnd += boost::gregorian::months(intervalValue);
break;
case TimeUnit::Day:
snappedTimeEnd += boost::gregorian::days(intervalValue);
break;
case TimeUnit::Hour:
snappedTimeEnd += boost::posix_time::hours(intervalValue);
break;
case TimeUnit::Minute:
snappedTimeEnd += boost::posix_time::minutes(intervalValue);
break;
case TimeUnit::Second:
snappedTimeEnd += boost::posix_time::seconds(intervalValue);
break;
}

time_start_mapping = boost::posix_time::to_time_t(snappedTimeStart);
time_end_mapping = boost::posix_time::to_time_t(snappedTimeEnd);

// format date to determine file
auto snappedTimeT = static_cast<time_t>(time_start_mapping);
tm snappedTimeTm = {};
gmtime_r(&snappedTimeT, &snappedTimeTm);

char date[MAX_FILE_NAME_LENGTH] = {0};
strftime(date, sizeof(date), time_format.c_str(), &snappedTimeTm);
std::string snappedTimeString(date);

std::string placeholder = "%%%TIME_STRING%%%";
size_t placeholderPos = fileName.find(placeholder);

fileName = fileName.replace(placeholderPos, placeholder.length(), snappedTimeString);
//only create the timesnap when a placeholder exists. else the filename is fixed.
if (placeholderPos != std::string::npos) {
Json::Value timeInterval = channelJson.get("time_interval", datasetJson.get("time_interval", Json::Value(
Json::objectValue)));
TimeUnit intervalUnit = GDALTimesnap::createTimeUnit(timeInterval.get("unit", "Month").asString());
int intervalValue = timeInterval.get("value", 1).asInt();


// doesn't work on older boost versions because seconds precision is limit to 32bit
// ptime start = from_time_t(static_cast<time_t>(time_start_mapping));
// ptime wanted = from_time_t(static_cast<time_t>(wantedTimeUnix));

ptime start = ptime(boost::gregorian::date(1970, 1, 1)) +
milliseconds(static_cast<long>(time_start_mapping * 1000));
ptime wanted =
ptime(boost::gregorian::date(1970, 1, 1)) + milliseconds(static_cast<long>(wantedTimeUnix * 1000));

//snap the time to the given interval
ptime snappedTimeStart = GDALTimesnap::snapToInterval(intervalUnit, intervalValue, start, wanted);

// snap end to the next interval
ptime snappedTimeEnd(snappedTimeStart);
switch (intervalUnit) {
case TimeUnit::Year:
snappedTimeEnd += boost::gregorian::years(intervalValue);
break;
case TimeUnit::Month:
snappedTimeEnd += boost::gregorian::months(intervalValue);
break;
case TimeUnit::Day:
snappedTimeEnd += boost::gregorian::days(intervalValue);
break;
case TimeUnit::Hour:
snappedTimeEnd += boost::posix_time::hours(intervalValue);
break;
case TimeUnit::Minute:
snappedTimeEnd += boost::posix_time::minutes(intervalValue);
break;
case TimeUnit::Second:
snappedTimeEnd += boost::posix_time::seconds(intervalValue);
break;
}

time_start_mapping = boost::posix_time::to_time_t(snappedTimeStart);
time_end_mapping = boost::posix_time::to_time_t(snappedTimeEnd);

// format date to determine file
auto snappedTimeT = static_cast<time_t>(time_start_mapping);
tm snappedTimeTm = {};
gmtime_r(&snappedTimeT, &snappedTimeTm);

char date[MAX_FILE_NAME_LENGTH] = {0};
strftime(date, sizeof(date), time_format.c_str(), &snappedTimeTm);
std::string snappedTimeString(date);

fileName = fileName.replace(placeholderPos, placeholder.length(), snappedTimeString);
}
}


Expand Down
41 changes: 32 additions & 9 deletions src/util/ogr_source_datasets.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

#include <unordered_set>
#include <boost/filesystem.hpp>
#include <boost/range/iterator_range.hpp>
#include "ogr_source_datasets.h"
Expand Down Expand Up @@ -91,6 +92,27 @@ Json::Value OGRSourceDatasets::getDatasetListing(const std::string &dataset_name
Json::Value textual(Json::ValueType::arrayValue);
Json::Value numeric(Json::ValueType::arrayValue);

//read the wanted attributes from the layer or dataset definition.
std::unordered_set<std::string> requested_attributes_numeric;
std::unordered_set<std::string> requested_attributes_textual;
requested_attributes_numeric.reserve(32);
requested_attributes_textual.reserve(32);
if(!columns_layer.isNull()){
for(auto &att : columns_layer["numeric"]){
requested_attributes_numeric.insert(att.asString());
}
for(auto &att : columns_layer["textual"]){
requested_attributes_textual.insert(att.asString());
}
} else {
for(auto &att : columns_dataset["numeric"]){
requested_attributes_numeric.insert(att.asString());
}
for(auto &att : columns_dataset["textual"]){
requested_attributes_textual.insert(att.asString());
}
}

OGRFeatureDefn *attributeDefn = layer->GetLayerDefn();

for(int j = 0; j < attributeDefn->GetFieldCount(); j++){
Expand All @@ -101,28 +123,29 @@ Json::Value OGRSourceDatasets::getDatasetListing(const std::string &dataset_name
{
//skip geometry attributes of csv files
if(field_name == getJsonParameter(columns_layer, columns_dataset, "x").asString() ||
hasJsonParameter(columns_layer, columns_dataset, "y") &&
field_name == getJsonParameter(columns_layer, columns_dataset, "y").asString())
(hasJsonParameter(columns_layer, columns_dataset, "y") &&
field_name == getJsonParameter(columns_layer, columns_dataset, "y").asString()))
{
continue;
}
}

//skip time attributes
if(hasJsonParameter(columns_layer, columns_dataset, "time1") &&
field_name == getJsonParameter(columns_layer, columns_dataset, "time1").asString() ||
hasJsonParameter(columns_layer, columns_dataset, "time2") &&
field_name == getJsonParameter(columns_layer, columns_dataset, "time2").asString())
if((hasJsonParameter(columns_layer, columns_dataset, "time1") &&
field_name == getJsonParameter(columns_layer, columns_dataset, "time1").asString()) ||
(hasJsonParameter(columns_layer, columns_dataset, "time2") &&
field_name == getJsonParameter(columns_layer, columns_dataset, "time2").asString()))
{
continue;
}

OGRFieldType type = field->GetType();

if(type == OFTInteger || type == OFTInteger64 || type == OFTReal)
//only add the attribute to available attributes if it is requested in dataset/layer.
if(requested_attributes_numeric.find(field_name) != requested_attributes_numeric.end()){
numeric.append(field_name);
else
} else if(requested_attributes_textual.find(field_name) != requested_attributes_textual.end()){
textual.append(field_name);
}
}

listing_json["textual"] = textual;
Expand Down
37 changes: 34 additions & 3 deletions test/systemtests/data/ogr_files/nyu.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,44 @@

"layers" : {
"sdr:nyu_2451_35023" : {
"time" : "none"
"time" : "none",
"columns" : {
"textual" : [
"exsdesc",
"f_code",
"f_codedesc"
],
"numeric" : [
"tile_id",
"end_id"
]
}
},
"sdr:nyu_2451_35659" : {
"time" : "none"
"time" : "none",
"columns" : {
"textual" : [
"f_code",
"f_codedesc",
"textstring"
],
"numeric" : [
"tile_id",
"txt_id"
]
}
},
"sdr:nyu_2451_38681" : {
"time" : "none"
"time" : "none",
"columns" : {
"textual" : [
"type",
"link"
],
"numeric" : [
"current"
]
}
}
}
}
27 changes: 22 additions & 5 deletions test/systemtests/data/ogr_files/ogr_source_dataset1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,34 @@
"on_error" : "keep",
"description" : "It's all about the cities",
"time": "start",
"duration" : 2678400,
"columns" : {
"x" : "wkt",
"time1" : "time_start",
"numeric" : [
"population"
],
"textual" : [
"name",
"is_in"
]
},
"duration" : 2678400,
"layers" : {
"osm_large_cities" : {
"osm_large_cities" : {
"columns" : {
"x" : "wkt",
"time1" : "time_start"
"time1" : "time_start",
"numeric" : [
"population"
],
"textual" : [
"name"
]
},
"time1_format" : {
"format" : "iso"
},
"geometry_type" : "points"
},
"geometry_type" : "points"
}
}
}
10 changes: 9 additions & 1 deletion test/systemtests/data/ogr_files/ogr_source_dataset2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,15 @@
"layers" : {
"ne_10m_ports" : {
"time" : "none",
"description" : "Ahoi"
"description" : "Ahoi",
"columns" : {
"textual" : [
"website"
],
"numeric" : [
"natlscale"
]
}
}
}
}