bes Updated for version 3.20.10
DmrppCommon.h
1
2// -*- mode: c++; c-basic-offset:4 -*-
3
4// This file is part of the BES
5
6// Copyright (c) 2016 OPeNDAP, Inc.
7// Author: James Gallagher <jgallagher@opendap.org>
8//
9// This library is free software; you can redistribute it and/or
10// modify it under the terms of the GNU Lesser General Public
11// License as published by the Free Software Foundation; either
12// version 2.1 of the License, or (at your option) any later version.
13//
14// This library is distributed in the hope that it will be useful,
15// but WITHOUT ANY WARRANTY; without even the implied warranty of
16// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17// Lesser General Public License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24
25#ifndef _dmrpp_common_h
26#define _dmrpp_common_h 1
27
28#include <string>
29#include <vector>
30#include <memory>
31
32#define PUGIXML_NO_XPATH
33#define PUGIXML_HEADER_ONLY
34#include <pugixml.hpp>
35
36namespace libdap {
37class DMR;
38class BaseType;
39class D4BaseTypeFactory;
40class D4Group;
41class D4Attributes;
42class D4EnumDef;
43class D4Dimension;
44class XMLWriter;
45}
46
47namespace http {
48class url;
49}
50
51namespace pugi {
52class xml_node;
53}
54
55namespace dmrpp {
56
57class DMZ;
58class Chunk;
59class DmrppArray;
60class SuperChunk;
61
62void join_threads(pthread_t threads[], unsigned int num_threads);
63
77
78 friend class DmrppCommonTest;
79 friend class DmrppParserTest;
80
81private:
82 bool d_compact = false;
83 std::string d_filters;
84 std::string d_byte_order;
85 std::vector<unsigned long long> d_chunk_dimension_sizes;
86 std::vector<std::shared_ptr<Chunk>> d_chunks;
87 bool d_twiddle_bytes = false;
88
89 // These indicate that the chunks or attributes have been loaded into the
90 // variable when the DMR++ handler is using lazy-loading of this data.
91 bool d_chunks_loaded = false;
92 bool d_attributes_loaded = false;
93
94 // Each instance of DmrppByte, ..., holds a shared pointer to the DMZ so that
95 // it can fetch more information from the XML if needed - this is how the lazy-load
96 // feature is implemented. The xml_node object is used to simplify finding where
97 // in the XML information about a variable is stored - to limit searching the
98 // document, the code caches the XML node.
99 std::shared_ptr<DMZ> d_dmz;
100 pugi::xml_node d_xml_node;
101
102protected:
103#if 0
106 virtual std::vector<std::shared_ptr<Chunk>> get_chunks() {
107 return d_chunks;
108 }
109#endif
110
111 virtual char *read_atomic(const std::string &name);
112
113 // This declaration allows code in the SuperChunky program to use the protected method.
114 // jhrg 10/25/21
115 friend void compute_super_chunks(dmrpp::DmrppArray *array, bool only_constrained, std::vector<dmrpp::SuperChunk *> &super_chunks);
116
117public:
118 static bool d_print_chunks;
119 static std::string d_dmrpp_ns;
120 static std::string d_ns_prefix;
121
122 DmrppCommon() = default;
123
124 DmrppCommon(std::shared_ptr<DMZ> dmz) : d_dmz(dmz) { }
125
126 DmrppCommon(const DmrppCommon &) = default;
127
128 virtual ~DmrppCommon()= default;
129
131 virtual std::string get_filters() const {
132 return d_filters;
133 }
134
135 void set_filter(const std::string &value);
136
137 virtual bool is_filters_empty() const {
138 return d_filters.empty();
139 }
140
142 virtual bool is_compact_layout() const {
143 return d_compact;
144 }
145
147 void set_compact(bool value) {
148 d_compact = value;
149 }
150
152 virtual bool twiddle_bytes() const { return d_twiddle_bytes; }
153
154 // @brief Provide access to the DMZ instance bound to this variable
155 // virtual const std::shared_ptr<DMZ> &get_dmz() const { return d_dmz; }
156
158 virtual bool get_chunks_loaded() const { return d_chunks_loaded; }
159 virtual void set_chunks_loaded(bool state) { d_chunks_loaded = state; }
160
162 virtual bool get_attributes_loaded() const { return d_attributes_loaded; }
163 virtual void set_attributes_loaded(bool state) { d_attributes_loaded = state; }
164
165 virtual const pugi::xml_node &get_xml_node() const { return d_xml_node; }
166 virtual void set_xml_node(pugi::xml_node node) { d_xml_node = node; }
167
169 virtual const std::vector<std::shared_ptr<Chunk>> &get_immutable_chunks() const {
170 return d_chunks;
171 }
172
175 virtual size_t get_chunks_size() const { return d_chunks.size(); }
176
179 virtual const std::vector<unsigned long long> &get_chunk_dimension_sizes() const {
180 return d_chunk_dimension_sizes;
181 }
182
185 virtual unsigned long long get_chunk_size_in_elements() const {
186 unsigned long long elements = 1;
187 for (auto d_chunk_dimension_size : d_chunk_dimension_sizes) {
188 elements *= d_chunk_dimension_size;
189 }
190
191 return elements;
192 }
193
194 void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space = "");
195
196 void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space = "", const std::string &encoded = "");
197
198 void print_dmrpp(libdap::XMLWriter &writer, bool constrained = false);
199
200 // Replaced hsize_t with size_t. This eliminates a dependency on hdf5. jhrg 9/7/18
202 void set_chunk_dimension_sizes(const std::vector<size_t> &chunk_dims) {
203 // tried using copy(chunk_dims.begin(), chunk_dims.end(), d_chunk_dimension_sizes.begin())
204 // it didn't work, maybe because of the differing element types?
205 for (auto chunk_dim : chunk_dims) {
206 d_chunk_dimension_sizes.push_back(chunk_dim);
207 }
208 }
209
210 // These two functions duplicate code in DMZ but provides access to the DMZ::load_chunks()
211 // method without having to cast a BaseType to a DmrppCommon in order to use it. jhrg 11/12/21
212 virtual void load_chunks(libdap::BaseType *btp);
213 virtual void load_attributes(libdap::BaseType *btp);
214
215 virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string);
216
217 virtual void ingest_compression_type(const std::string &compression_type_string);
218
219 virtual void ingest_byte_order(const std::string &byte_order_string);
220 virtual std::string get_byte_order() const { return d_byte_order; }
221
222 virtual unsigned long add_chunk(
223 std::shared_ptr<http::url> d_data_url,
224 const std::string &byte_order,
225 unsigned long long size,
226 unsigned long long offset,
227 const std::string &position_in_array);
228
229 virtual unsigned long add_chunk(
230 std::shared_ptr<http::url> d_data_url,
231 const std::string &byte_order,
232 unsigned long long size,
233 unsigned long long offset,
234 const std::vector<unsigned long long> &position_in_array);
235
236 virtual unsigned long add_chunk(
237 const std::string &byte_order,
238 unsigned long long size,
239 unsigned long long offset,
240 const std::string &position_in_array);
241
242 virtual unsigned long add_chunk(
243 const std::string &byte_order,
244 unsigned long long size,
245 unsigned long long offset,
246 const std::vector<unsigned long long> &position_in_array);
247
248 virtual void dump(std::ostream & strm) const;
249};
250
251} // namespace dmrpp
252
253#endif // _dmrpp_common_h
254
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
Size and offset information of data included in DMR++ files.
Definition: DmrppCommon.h:76
static std::string d_ns_prefix
The XML namespace prefix to use.
Definition: DmrppCommon.h:120
virtual bool twiddle_bytes() const
Returns true if this object utilizes shuffle compression.
Definition: DmrppCommon.h:152
void set_chunk_dimension_sizes(const std::vector< size_t > &chunk_dims)
Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.
Definition: DmrppCommon.h:202
static bool d_print_chunks
if true, print_dap4() prints chunk elements
Definition: DmrppCommon.h:118
virtual bool is_compact_layout() const
Returns true if this object utilizes COMPACT layout.
Definition: DmrppCommon.h:142
virtual void ingest_compression_type(const std::string &compression_type_string)
Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned i...
Definition: DmrppCommon.cc:171
virtual void load_attributes(libdap::BaseType *btp)
Load the attribute information for this variable.
Definition: DmrppCommon.cc:491
void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="")
Print the Compact base64-encoded information.
Definition: DmrppCommon.cc:404
virtual bool get_chunks_loaded() const
Have the chunks been loaded?
Definition: DmrppCommon.h:158
virtual size_t get_chunks_size() const
Use this when the number of chunks is needed.
Definition: DmrppCommon.h:175
static std::string d_dmrpp_ns
The DMR++ XML namespace.
Definition: DmrppCommon.h:119
void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space="")
Print the Chunk information.
Definition: DmrppCommon.cc:330
virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string)
Set the dimension sizes for a chunk.
Definition: DmrppCommon.cc:134
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
Definition: DmrppCommon.h:169
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Add a new chunk as defined by an h4:byteStream element.
Definition: DmrppCommon.cc:204
void set_filter(const std::string &value)
Set the value of the filters property.
Definition: DmrppCommon.cc:108
virtual void ingest_byte_order(const std::string &byte_order_string)
Parses the text content of the XML element chunks:byteOrder.
Definition: DmrppCommon.cc:182
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
Definition: DmrppCommon.h:179
void set_compact(bool value)
Set the value of the compact property.
Definition: DmrppCommon.h:147
void print_dmrpp(libdap::XMLWriter &writer, bool constrained=false)
Print the DMR++ response for the Scalar types.
Definition: DmrppCommon.cc:427
virtual void load_chunks(libdap::BaseType *btp)
Load chunk information for this variable.
Definition: DmrppCommon.cc:482
virtual unsigned long long get_chunk_size_in_elements() const
Get the number of elements in this chunk.
Definition: DmrppCommon.h:185
virtual std::string get_filters() const
Return the names of all the filters in the order they were applied.
Definition: DmrppCommon.h:131
virtual bool get_attributes_loaded() const
Have the attributes been loaded?
Definition: DmrppCommon.h:162
virtual char * read_atomic(const std::string &name)
read method for the atomic types
Definition: DmrppCommon.cc:311
utility class for the HTTP catalog module
Definition: AllowedHosts.cc:55