bes Updated for version 3.20.10
NCMLRequestHandler.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2009 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29#include "config.h"
30
31#include <memory>
32
33#include <libdap/DMR.h>
34#include <libdap/DataDDS.h>
35
36#include <libdap/mime_util.h>
37#include <libdap/D4BaseTypeFactory.h>
38
39#include "NCMLRequestHandler.h"
40
41#include <BESConstraintFuncs.h>
42#include <BESContainerStorage.h>
43#include <BESContainerStorageList.h>
44#include <BESDapNames.h>
45#include "BESDataDDSResponse.h"
46#include <BESDataNames.h>
47#include <BESDASResponse.h>
48#include <BESDDSResponse.h>
49#include <BESDMRResponse.h>
50
51#include <BESDebug.h>
52#include "BESStopWatch.h"
53#include <BESInternalError.h>
54#include <BESDapError.h>
55#include <BESError.h>
56#include <BESRequestHandlerList.h>
57#include <BESResponseHandler.h>
58#include <BESResponseNames.h>
59#include <BESServiceRegistry.h>
60#include <BESTextInfo.h>
61#include <BESUtil.h>
62#include <BESVersionInfo.h>
63#include <TheBESKeys.h>
64
65#include "DDSLoader.h"
66
67#include "NCMLDebug.h"
68#include "NCMLUtil.h"
69#include "NCMLParser.h"
70#include "NCMLResponseNames.h"
71#include "SimpleLocationParser.h"
72
73using namespace agg_util;
74using namespace ncml_module;
75using namespace libdap;
76
77bool NCMLRequestHandler::_global_attributes_container_name_set = false;
78string NCMLRequestHandler::_global_attributes_container_name = "";
79
80NCMLRequestHandler::NCMLRequestHandler(const string &name) :
82{
83 add_method(DAS_RESPONSE, NCMLRequestHandler::ncml_build_das);
84 add_method(DDS_RESPONSE, NCMLRequestHandler::ncml_build_dds);
85 add_method(DATA_RESPONSE, NCMLRequestHandler::ncml_build_data);
86
87 add_method(DMR_RESPONSE, NCMLRequestHandler::ncml_build_dmr);
88 add_method(DAP4DATA_RESPONSE, NCMLRequestHandler::ncml_build_dmr);
89
90 add_method(VERS_RESPONSE, NCMLRequestHandler::ncml_build_vers);
91 add_method(HELP_RESPONSE, NCMLRequestHandler::ncml_build_help);
92
93 if (NCMLRequestHandler::_global_attributes_container_name_set == false) {
94 bool key_found = false;
95 string value;
96 TheBESKeys::TheKeys()->get_value("NCML.GlobalAttributesContainerName", value, key_found);
97 if (key_found) {
98 // It was set in the conf file
99 NCMLRequestHandler::_global_attributes_container_name_set = true;
100
101 NCMLRequestHandler::_global_attributes_container_name = value;
102 }
103 }
104}
105
106NCMLRequestHandler::~NCMLRequestHandler()
107{
108}
109
110#if 0
111// Not used. jhrg 4/16/14
112
113// This is the original example from Patrick or James for loading local file within the BES...
114// Still used by DataDDS call, but the other callbacks use DDSLoader
115// to get a brandy new DDX response.
116// @see DDSLoader
117bool NCMLRequestHandler::ncml_build_redirect(BESDataHandlerInterface &dhi, const string& location)
118{
119 // The current container in dhi is a reference to the ncml file.
120 // Need to parse the ncml file here and get the list of locations
121 // that we will be using. Any constraints defined?
122
123 // do this for each of the locations retrieved from the ncml file.
124 // If there are more than one locations in the ncml then we can't
125 // set the context for dap_format to dap2. This will create a
126 // structure for each of the locations in the resulting dap object.
127 string sym_name = dhi.container->get_symbolic_name();
128 BESContainerStorageList *store_list = BESContainerStorageList::TheList();
129 BESContainerStorage *store = store_list->find_persistence("catalog");
130 if (!store) {
131 throw BESInternalError("couldn't find the catalog storage", __FILE__, __LINE__);
132 }
133 // this will throw an exception if the location isn't found in the
134 // catalog. Might want to catch this. Wish the add returned the
135 // container object created. Might want to change it.
136 string new_sym = sym_name + "_location1";
137 store->add_container(new_sym, location, "");
138
139 BESContainer *container = store->look_for(new_sym);
140 if (!container) {
141 throw BESInternalError("couldn't find the container" + sym_name, __FILE__, __LINE__);
142 }
143 BESContainer *ncml_container = dhi.container;
144 dhi.container = container;
145
146 // this will throw an exception if there is a problem building the
147 // response for this container. Might want to catch this
148 BESRequestHandlerList::TheList()->execute_current(dhi);
149
150 // clean up
151 dhi.container = ncml_container;
152 store->del_container(new_sym);
153
154 return true;
155}
156#endif
157
158// Here we load the DDX response with by hijacking the current dhi via DDSLoader
159// and hand it to our parser to load the ncml, load the DDX for the location,
160// apply ncml transformations to it, then return the modified DDS.
161bool NCMLRequestHandler::ncml_build_das(BESDataHandlerInterface &dhi)
162{
163 BESStopWatch sw;
164 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_das", dhi.data[REQUEST_ID]);
165
166 string filename = dhi.container->access();
167
168 // Any exceptions winding through here will cause the loader and parser dtors
169 // to clean up dhi state, etc.
170 DDSLoader loader(dhi);
171 NCMLParser parser(loader);
172 auto_ptr<BESDapResponse> loaded_bdds = parser.parse(filename, DDSLoader::eRT_RequestDDX);
173
174 // Now fill in the desired DAS response object from the DDS
175 DDS* dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
176 VALID_PTR(dds);
177
178 BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(dhi.response_handler->get_response_object());
179 VALID_PTR(bdas);
180
181 // Copy the modified DDS attributes into the DAS response object!
182 DAS *das = bdas->get_das();
183
184 if (dds->get_dap_major() < 4)
185 NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
186 NCMLRequestHandler::get_global_attributes_container_name());
187
189
190 // loaded_bdds destroys itself.
191 return true;
192}
193
194bool NCMLRequestHandler::ncml_build_dds(BESDataHandlerInterface &dhi)
195{
196#if 0
197 // original version 8/13/15
198 BESStopWatch sw;
199 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_dds", dhi.data[REQUEST_ID]);
200
201 string filename = dhi.container->access();
202
203 // Any exceptions winding through here will cause the loader and parser dtors
204 // to clean up dhi state, etc.
205 auto_ptr<BESDapResponse> loaded_bdds(0);
206 {
207 DDSLoader loader(dhi);
208 NCMLParser parser(loader);
209 loaded_bdds = parser.parse(filename, DDSLoader::eRT_RequestDDX);
210 }
211 if (!loaded_bdds.get()) {
212 throw BESInternalError("Null BESDDSResonse in ncml DDS handler.", __FILE__, __LINE__);
213 }
214
215 // Poke the handed down original response object with the loaded and modified one.
216 DDS* dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
217 VALID_PTR(dds);
218 BESResponseObject *response = dhi.response_handler->get_response_object();
219 BESDDSResponse *bdds_out = dynamic_cast<BESDDSResponse *>(response);
220 VALID_PTR(bdds_out);
221 DDS *dds = bdds_out->get_dds();
222 VALID_PTR(dds);
223
224 if (dds->get_dap_major() < 4)
225 NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
226 NCMLRequestHandler::get_global_attributes_container_name());
227
228 // If we just use DDS::operator=, we get into trouble with copied
229 // pointers, bashing of the dataset name, etc etc so I specialize the copy for now.
231
232 // Apply constraints to the result
233 // See comment below. jhrg 8/12/15 dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
234 bdds_out->set_constraint(dhi);
235
236 // Also copy in the name of the original ncml request
237 // TODO @HACK Not sure I want just the basename for the filename,
238 // but since the DDS/DataDDS response fills the dataset name with it,
239 // Our bes-testsuite fails since we get local path info in the dataset name.
240 dds->filename(name_path(filename));
241 dds->set_dataset_name(name_path(filename));
242
243 return true;
244#endif
245
246 BESStopWatch sw;
247 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_dds", dhi.data[REQUEST_ID]);
248
249 string filename = dhi.container->access();
250
251 // it better be a data response!
252 BESDDSResponse* ddsResponse = dynamic_cast<BESDDSResponse *>(dhi.response_handler->get_response_object());
253 NCML_ASSERT_MSG(ddsResponse,
254 "NCMLRequestHandler::ncml_build_data(): expected BESDDSResponse* but didn't get it!!");
255
256 // Block it up to force cleanup of DHI.
257 {
258 DDSLoader loader(dhi);
259 NCMLParser parser(loader);
260 parser.parseInto(filename, DDSLoader::eRT_RequestDDX, ddsResponse);
261 }
262
263 DDS *dds = ddsResponse->get_dds();
264 VALID_PTR(dds);
265
266 if (dds->get_dap_major() < 4)
267 NCMLUtil::hackGlobalAttributesForDAP2(dds->get_attr_table(),
268 NCMLRequestHandler::get_global_attributes_container_name());
269
270 // Apply constraints to the result
271 // See comment below. jhrg 8/12/15 dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
272 ddsResponse->set_constraint(dhi);
273
274 // Also copy in the name of the original ncml request
275 dds->filename(name_path(filename));
276 dds->set_dataset_name(name_path(filename));
277
278 return true;
279}
280
281bool NCMLRequestHandler::ncml_build_data(BESDataHandlerInterface &dhi)
282{
283 BESStopWatch sw;
284 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_data", dhi.data[REQUEST_ID]);
285
286 string filename = dhi.container->access();
287
288 // it better be a data response!
289 BESDataDDSResponse* dataResponse = dynamic_cast<BESDataDDSResponse *>(dhi.response_handler->get_response_object());
290 NCML_ASSERT_MSG(dataResponse,
291 "NCMLRequestHandler::ncml_build_data(): expected BESDataDDSResponse* but didn't get it!!");
292
293 // Block it up to force cleanup of DHI.
294 {
295 DDSLoader loader(dhi);
296 NCMLParser parser(loader);
297 parser.parseInto(filename, DDSLoader::eRT_RequestDataDDS, dataResponse);
298 }
299
300 // Apply constraints to the result
301
302 // dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();
303 // Replaced the above with the code below. P West said, a while ago, that using set_constraint
304 // was better because BES containers would be supported. Not sure if that's a factor in this
305 // code... jhrg 8/12/15
306 dataResponse->set_constraint(dhi);
307
308 // Also copy in the name of the original ncml request
309 DDS* dds = NCMLUtil::getDDSFromEitherResponse(dataResponse);
310 VALID_PTR(dds);
311
312 dds->filename(name_path(filename));
313 dds->set_dataset_name(name_path(filename));
314
315 return true;
316}
317
318bool NCMLRequestHandler::ncml_build_dmr(BESDataHandlerInterface &dhi)
319{
320 BESStopWatch sw;
321 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("NCMLRequestHandler::ncml_build_dmr", dhi.data[REQUEST_ID]);
322
323 // Because this code does not yet know how to build a DMR directly, use
324 // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
325 // First step, build the 'full DDS'
326 string data_path = dhi.container->access();
327
328 DDS *dds = 0; // This will be deleted when loaded_bdds goes out of scope.
329 auto_ptr<BESDapResponse> loaded_bdds(0);
330 try {
331 DDSLoader loader(dhi);
332 NCMLParser parser(loader);
333 loaded_bdds = parser.parse(data_path, DDSLoader::eRT_RequestDDX);
334 if (!loaded_bdds.get()) throw BESInternalError("Null BESDDSResonse in ncml DDS handler.", __FILE__, __LINE__);
335 dds = NCMLUtil::getDDSFromEitherResponse(loaded_bdds.get());
336 VALID_PTR(dds);
337 dds->filename(data_path);
338 dds->set_dataset_name(data_path);
339 }
340 catch (InternalErr &e) {
341 throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
342 }
343 catch (Error &e) {
344 throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
345 }
346 catch (BESError &e){
347 throw;
348 }
349 catch (...) {
350 throw BESDapError("Caught unknown error build ** DMR response", true, unknown_error, __FILE__, __LINE__);
351 }
352
353 // Extract the DMR Response object - this holds the DMR used by the
354 // other parts of the framework.
355 BESResponseObject *response = dhi.response_handler->get_response_object();
356 BESDMRResponse &bdmr = dynamic_cast<BESDMRResponse &>(*response);
357
358 // Get the DMR made by the BES in the BES/dap/BESDMRResponseHandler, make sure there's a
359 // factory we can use and then dump the DAP2 variables and attributes in using the
360 // BaseType::transform_to_dap4() method that transforms individual variables
361 DMR *dmr = bdmr.get_dmr();
362 dmr->set_factory(new D4BaseTypeFactory);
363 dmr->build_using_dds(*dds);
364
365 // Instead of fiddling with the internal storage of the DHI object,
366 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
367 // methods to set the constraints. But, why? Ans: from Patrick is that
368 // in the 'container' mode of BES each container can have a different
369 // CE.
370 bdmr.set_dap4_constraint(dhi);
371 bdmr.set_dap4_function(dhi);
372
373 return true;
374}
375
376bool NCMLRequestHandler::ncml_build_vers(BESDataHandlerInterface &dhi)
377{
378 BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
379 if (!info) throw InternalErr(__FILE__, __LINE__, "Expected a BESVersionInfo instance");
380
381 info->add_module(MODULE_NAME, MODULE_VERSION);
382 return true;
383}
384
385bool NCMLRequestHandler::ncml_build_help(BESDataHandlerInterface &dhi)
386{
387 BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
388 if (!info) throw InternalErr(__FILE__, __LINE__, "Expected a BESVersionInfo instance");
389
390 // This is an example. If you had a help file you could load it like
391 // this and if your handler handled the following responses.
392 map<string, string> attrs;
393 attrs["name"] = MODULE_NAME;
394 attrs["version"] = MODULE_VERSION;
395
396 list<string> services;
397 BESServiceRegistry::TheRegistry()->services_handled(ncml_module::ModuleConstants::NCML_NAME, services);
398 if (services.size() > 0) {
399 string handles = BESUtil::implode(services, ',');
400 attrs["handles"] = handles;
401 }
402 info->begin_tag("module", &attrs);
403 //info->add_data_from_file( "NCML.Help", "NCML Help" ) ;
404 info->add_data("Please consult the online documentation at " + ncml_module::ModuleConstants::DOC_WIKI_URL);
405 info->end_tag("module");
406
407 return true;
408}
409
410void NCMLRequestHandler::dump(ostream &strm) const
411{
412 strm << BESIndent::LMarg << "NCMLRequestHandler::dump - (" << (void *) this << ")" << endl;
413 BESIndent::Indent();
415 BESIndent::UnIndent();
416}
417
Provides a mechanism for accessing container information from different container stores registered w...
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
provides persistent storage for data storage information represented by a container.
virtual bool del_container(const std::string &s_name)=0
removes a container with the given symbolic name
virtual void add_container(const std::string &sym_name, const std::string &real_name, const std::string &type)=0
adds a container with the provided information
virtual BESContainer * look_for(const std::string &sym_name)=0
looks for a container in this persistent store
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
Holds a DDS object within the BES.
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
informational response object
Definition: BESInfo.h:63
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
exception thrown if internal error encountered
virtual void execute_current(BESDataHandlerInterface &dhi)
Execute a single method for the current container that will fill in the response object rather than i...
Represents a specific data type request handler.
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:657
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
virtual void dump(std::ostream &strm) const
dumps information about this object
static libdap::DDS * getDDSFromEitherResponse(BESDapResponse *response)
Definition: NCMLUtil.cc:356
static void populateDASFromDDS(libdap::DAS *das, const libdap::DDS &dds_const)
Definition: NCMLUtil.cc:276
static void copyVariablesAndAttributesInto(libdap::DDS *dds_out, const libdap::DDS &dds_in)
Definition: NCMLUtil.cc:332
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...
static const std::string DOC_WIKI_URL
static const std::string NCML_NAME