dg::file::SerialNcFile Struct Reference

Serial NetCDF-4 file. More...

Public Types
using	Hyperslab = NcHyperslab

Public Member Functions
	SerialNcFile ()=default
	SerialNcFile (const std::filesystem::path &filename, enum NcFileMode mode=nc_nowrite)
	Open/Create a netCDF file.
	SerialNcFile (const SerialNcFile &rhs)=delete
	There can only be exactly one file handle per physical file.
SerialNcFile &	operator= (const SerialNcFile &rhs)=delete
	There can only be exactly one file handle per physical file.
	SerialNcFile (SerialNcFile &&rhs)=default
	Swap resources between two file handles.
SerialNcFile &	operator= (SerialNcFile &&rhs)=default
	Swap resources between two file handles.
	~SerialNcFile ()
	Close open nc file and release all resources.
void	open (const std::filesystem::path &filename, enum NcFileMode mode=nc_nowrite)
bool	is_open () const noexcept
void	close ()
	Explicitly close a file.
void	sync ()
	Call nc_sync.
int	get_ncid () const noexcept
	Get the ncid of the underlying NetCDF C-API.
void	def_grp (std::string name)
void	def_grp_p (std::filesystem::path path)
bool	grp_is_defined (std::filesystem::path path) const
	Check for existence of the group given by path.
void	set_grp (std::filesystem::path path="")
	Change group to path.
void	rename_grp (std::string old_name, std::string new_name)
int	get_grpid () const noexcept
	Get the NetCDF-C ID of the current group.
std::filesystem::path	get_current_path () const
	Get the absolute path of the current group.
std::list< std::filesystem::path >	get_grps () const
std::list< std::filesystem::path >	get_grps_r () const
	Get all subgroups recursively in the current group as absolute paths.
void	def_dim (std::string name, size_t size)
	Define a dimension named name of size size.
void	rename_dim (std::string old_name, std::string new_name)
	Rename a dimension from old_name to new_name.
size_t	get_dim_size (std::string name) const
	Get the size of the dimension named name.
std::vector< size_t >	get_dims_shape (const std::vector< std::string > &dims) const
	Get the size of each dimension in dims.
std::vector< std::string >	get_dims (bool include_parents=true) const
std::vector< std::string >	get_unlim_dims () const
bool	dim_is_defined (std::string name) const
void	put_att (const std::pair< std::string, nc_att_t > &att, std::string id="")
	Put an individual attribute.
template<class S , class T >
void	put_att (const std::tuple< S, nc_type, T > &att, std::string id="")
	Put an individual attribute of preset type to variable id.
template<class Attributes = std::map<std::string, nc_att_t>>
void	put_atts (const Attributes &atts, std::string id="")
	Write a collection of attributes to a NetCDF variable or file.
template<class T >
T	get_att_as (std::string att_name, std::string id="") const
	Get an attribute named att_name of the group or variable id.
template<class T >
std::vector< T >	get_att_vec_as (std::string att_name, std::string id="") const
	Short for get_att_as<std::vector<T>>( id, att_name);
template<class T >
std::map< std::string, T >	get_atts_as (std::string id="") const
	Read all NetCDF attributes of a certain type.
std::map< std::string, nc_att_t >	get_atts (std::string id="") const
void	del_att (std::string att_name, std::string id="")
	Remove an attribute named att_name from variable id.
bool	att_is_defined (std::string att_name, std::string id="") const
void	rename_att (std::string old_att_name, std::string new_att_name, std::string id="")
template<class T , class Attributes = std::map<std::string, nc_att_t>>
void	def_var_as (std::string name, const std::vector< std::string > &dim_names, const Attributes &atts={})
	Define a variable with given type, dimensions and (optionally) attributes.
template<class Attributes = std::map<std::string, nc_att_t>>
void	def_var (std::string name, nc_type xtype, const std::vector< std::string > &dim_names, const Attributes &atts={})
	Define a variable with given type, dimensions and (optionally) attributes.
template<class ContainerType , std::enable_if_t< dg::is_vector_v< ContainerType, SharedVectorTag >, bool > = true>
void	put_var (std::string name, const NcHyperslab &slab, const ContainerType &data)
	Write data to a variable.
template<class ContainerType , class Attributes = std::map<std::string, nc_att_t>, std::enable_if_t< dg::is_vector_v< ContainerType, SharedVectorTag >, bool > = true>
void	defput_var (std::string name, const std::vector< std::string > &dim_names, const Attributes &atts, const NcHyperslab &slab, const ContainerType &data)
	Define and put a variable in one go.
template<class T , std::enable_if_t< dg::is_scalar_v< T >, bool > = true>
void	put_var (std::string name, const std::vector< size_t > &start, T data)
	Write a single data point.
template<class T , class Attributes = std::map<std::string, nc_att_t>>
void	def_dimvar_as (std::string name, size_t size, const Attributes &atts)
	Define a dimension and dimension variable in one go.
template<class ContainerType , class Attributes = std::map<std::string, nc_att_t>>
void	defput_dim (std::string name, const Attributes &atts, const ContainerType &abscissas)
	Define a dimension and define and write to a dimension variable in one go.
template<class ContainerType , std::enable_if_t< dg::is_vector_v< ContainerType, SharedVectorTag >, bool > = true>
void	get_var (std::string name, const NcHyperslab &slab, ContainerType &data) const
	Read hyperslab slab from variable named name into container data.
template<class T , std::enable_if_t< dg::is_scalar_v< T >, bool > = true>
void	get_var (std::string name, const std::vector< size_t > &start, T &data) const
	Read scalar from position start from variable named name.
bool	var_is_defined (std::string name) const
nc_type	get_var_type (std::string name) const
std::vector< std::string >	get_var_dims (std::string name) const
	Get the dimension names associated to variable name.
std::list< std::string >	get_var_names () const
	Get a list of variable names in the current group.

Detailed Description

Serial NetCDF-4 file.

Our take on a modern C++ implementation of the NetCDF-4 data model

See here a usage example

    // This example compiles in both serial and MPI environments
#ifdef WITH_MPI
    MPI_Comm comm = dg::mpi_cart_create( MPI_COMM_WORLD, {0,0}, {1, 1});
#endif
    // Open file and put some attributes
    dg::file::NcFile file( "test.nc", dg::file::nc_clobber);
    file.put_att( {"title", "Hello world"});
    file.put_att( {"truth", 42});
 
    // Generate a grid
    const double x0 = 0., x1 = 2.*M_PI;
    dg::x::CartesianGrid2d grid( x0,x1,x0,x1,3,10,10
#ifdef WITH_MPI
    , comm
#endif
    );
    // and put dimensions to file
    file.defput_dim( "x", {{"axis", "X"},
        {"long_name", "x-coordinate in Cartesian system"}},
        grid.abscissas(0));
    file.defput_dim( "y", {{"axis", "Y"},
        {"long_name", "y-coordinate in Cartesian system"}},
        grid.abscissas(1));
    // Generate some data and write to file
    dg::x::HVec data = dg::evaluate( function, grid);
    // Defne and write a variable in one go
    file.defput_var( "variable", {"y", "x"},
                {{"long_name", "A long explanation"}, {"unit", "m/s"}},
                grid, data);
 
    // Generate an unlimited dimension and define another variable
    file.def_dimvar_as<double>( "time", NC_UNLIMITED, {{"axis", "T"}});
    file.def_var_as<double>( "dependent", {"time", "y", "x"},
        {{"long_name", "Really interesting"}});
    // Write timeseries
    for( unsigned u=0; u<=2; u++)
    {
        double time = u*0.01;
        file.put_var("time", {u}, time);
        // We can write directly from GPU
        dg::x::DVec data = dg::evaluate( function, grid);
        dg::blas1::scal( data, cos(time));
        file.put_var( "dependent", {u, grid}, data);
    }
    file.close();
 
    // Open file for reading
    file.open( "test.nc", dg::file::nc_nowrite);
    std::string title = file.get_att_as<std::string>( "title");
    // In MPI all ranks automatically get the right chunk of data
    file.get_var( "variable", grid, data);
    unsigned NT = file.get_dim_size( "time");
    CHECK( NT == 3);
    double time;
    file.get_var( "time", {0}, time);
    file.close();

Note

This is a singleton that cannot be copied/assigned but only moved/move-assign

Only Netcdf-4 files are supported

The class hides all integer ids that the NetCDF C-library uses ("Ids do not exist in the Netcdf-4 data model!")

Most member functions will throw if they are called on a closed file

See also

Conventions to follow are the CF-conventions and netCDF conventions

Member Typedef Documentation

Hyperslab

using dg::file::SerialNcFile::Hyperslab = NcHyperslab

Constructor & Destructor Documentation

SerialNcFile() [1/4]

dg::file::SerialNcFile::SerialNcFile ( )

default

Construct a File Handle not associated to any file

dg::file::NcFile file;
file.open("test.nc", dg::file::nc_noclobber);
CHECK( file.is_open());
file.close();

SerialNcFile() [2/4]

dg::file::SerialNcFile::SerialNcFile ( const std::filesystem::path & filename, enum NcFileMode mode = nc_nowrite )

inline

Open/Create a netCDF file.

Parameters

filename	Name or path including the name of the netCDF file to open or create. The path may be either absolute or relative to the execution path of the program i.e. relative to std::filesystem::current_path()
mode	(see NcFileMode for nc_nowrite, nc_write, nc_clobber, nc_noclobber)

dg::file::NcFile file( "../test.nc", dg::file::nc_clobber);
CHECK( std::filesystem::exists( "../test.nc"));
CHECK( file.is_open());
// Cannot open another file while open
CHECK_THROWS_AS( file.open("test.nc"), dg::file::NC_Error);
file.close();

See also

NcFileMode

Here is the call graph for this function:

SerialNcFile() [3/4]

dg::file::SerialNcFile::SerialNcFile ( const SerialNcFile & rhs )

delete

There can only be exactly one file handle per physical file.

The reason is that the destructor releases all resources and thus a copy of the file that is subsequently destroyed leaves the original in an invalid state

SerialNcFile() [4/4]

dg::file::SerialNcFile::SerialNcFile ( SerialNcFile && rhs )

default

Swap resources between two file handles.

~SerialNcFile()

dg::file::SerialNcFile::~SerialNcFile ( )

inline

Close open nc file and release all resources.

Note

A destructor never throws any errors and we will just print a warning to std::cerr if something goes wrong

Here is the call graph for this function:

Member Function Documentation

att_is_defined()

bool dg::file::SerialNcFile::att_is_defined ( std::string att_name, std::string id = "" ) const

inline

Check for existence of the attribute named att_name in variable id

file.put_att({"same", "thing"});
CHECK( file.att_is_defined( "same"));
file.del_att( "same", "");
CHECK( not file.att_is_defined( "same"));

close()

void dg::file::SerialNcFile::close ( )

inline

Explicitly close a file.

Closing a file triggers all buffers to be written and memory to be released. After closing a new file can be associated to this handle again.

dg::file::NcFile file;
file.open("test.nc", dg::file::nc_noclobber);
CHECK( file.is_open());
file.close();

Note

This function may throw

def_dim()

void dg::file::SerialNcFile::def_dim ( std::string name, size_t size )

inline

Define a dimension named name of size size.

Note

Remember that dimensions do not have attributes or types, only variables and groups have attributes and only variables have types

One often defines an associated dimension variable with the same name as the dimension.

Parameters

name	of the dimension. Cannot be the same name as a dimension name already existing in the current group
size	Size of the dimension to create. Use NC_UNLIMITED to create an unlimited dimension

Note

There are a couple of subtle issues related to unlimited dimensions in NetCDF. First of all, all variables that share a dimension in NetCDF have the same size along that dimension. Second, it is possible to write data to any index (the start value of the hyperslab in the unlimited dimension can be anything) of a variable with an unlimited dimension. For example you can do

file.def_dim( "time", NC_UNLIMITED);
file.def_var_as<double>( "var", {"time"}, {});
file.put_var( "var", {5}, 42); // perfectly legal

The size of an unlimited dimension is the maximum of the sizes of all variables that share this dimension. If you are trying to read a variable at a slice where data was never written, the NetCDF library just fills it up with Fill Values. However, trying to read beyond the size of the unlimited dimension will fail. If a user wants to keep all unlimited variables synchronised, they unfortunately have to keep track of which variable was written themselves: See NetCDF issue

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group.

def_dimvar_as()

template<class T , class Attributes = std::map<std::string, nc_att_t>>

void dg::file::SerialNcFile::def_dimvar_as ( std::string name, size_t size, const Attributes & atts )

inline

Define a dimension and dimension variable in one go.

Short for

def_dim( name, size);
def_var_as<T>( name, {name}, atts);

Parameters

name	Name of the dimension and associated dimension variable
size	Size of the dimension to create. Use NC_UNLIMITED to create an unlimited dimension
atts	Suggested attribute is "axis" : "T" which enable paraview to recognize the dimension as the time axis

Note

This function is mainly intended to define an unlimited dimension as in

file.def_dimvar_as<double>( "time", NC_UNLIMITED, {{"axis", "T"}});
file.def_var_as<double>( "Energy", {"time"}, {{"long_name", "Energy"}});
for( unsigned i=0; i<=6; i++)
{
    file.put_var("time", {i}, i);
    if( i%2 == 0)
        file.put_var( "Energy", {i}, i);
}

Note

There are a couple of subtle issues related to unlimited dimensions in NetCDF. First of all, all variables that share a dimension in NetCDF have the same size along that dimension. Second, it is possible to write data to any index (the start value of the hyperslab in the unlimited dimension can be anything) of a variable with an unlimited dimension. For example you can do

file.def_dim( "time", NC_UNLIMITED);
file.def_var_as<double>( "var", {"time"}, {});
file.put_var( "var", {5}, 42); // perfectly legal

The size of an unlimited dimension is the maximum of the sizes of all variables that share this dimension. If you are trying to read a variable at a slice where data was never written, the NetCDF library just fills it up with Fill Values. However, trying to read beyond the size of the unlimited dimension will fail. If a user wants to keep all unlimited variables synchronised, they unfortunately have to keep track of which variable was written themselves: See NetCDF issue

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group.

Here is the call graph for this function:

def_grp()

void dg::file::SerialNcFile::def_grp ( std::string name )

inline

Define a group named name in the current group

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

Think of this as the bash command mkdir name

Parameters

name	of the new group

Note

Just like in a filesystem 2 nested groups with same name can exist but not 2 groups with the same name in the same group.

def_grp_p()

void dg::file::SerialNcFile::def_grp_p ( std::filesystem::path path )

inline

Define a group named path and all required intermediary groups

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

Think of this as the bash command mkdir -p path

Parameters

path	of the new group. Can be absolute or relative to the current group

Note

Just like in a filesystem 2 nested groups with same name can exist but not 2 groups with the same name in the same group.

Here is the call graph for this function:

def_var()

template<class Attributes = std::map<std::string, nc_att_t>>

void dg::file::SerialNcFile::def_var ( std::string name, nc_type xtype, const std::vector< std::string > & dim_names, const Attributes & atts = {} )

inline

Define a variable with given type, dimensions and (optionally) attributes.

Parameters

name	Name of the variable to define
xtype	NetCDF typeid
dim_names	Names of visible dimensions in the current group Can be empty which makes the defined variable a scalar w/o dimensions.

Attention

When writing variables, NetCDF-C always assumes that the last dimension of the NetCDF variable varies fastest in the given array. This is in contrast to the default behaviour of our dg::evaluate function, which produces vectors where the first dimension of the given grid varies fastest. Thus, when defining variable dimensions the dimension name of the first grid dimension needs to come last.

Note

The unlimited dimension, if present, must be the first dimension.

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group.

Parameters

atts	Attributes to put for the variable

Note

This function overload is useful if you want to use a compound type

def_var_as()

template<class T , class Attributes = std::map<std::string, nc_att_t>>

void dg::file::SerialNcFile::def_var_as ( std::string name, const std::vector< std::string > & dim_names, const Attributes & atts = {} )

inline

Define a variable with given type, dimensions and (optionally) attributes.

Parameters

name	Name of the variable to define
dim_names	Names of visible dimensions in the current group. Can be empty which makes the defined variable a scalar w/o dimensions.

Attention

When writing variables, NetCDF-C always assumes that the last dimension of the NetCDF variable varies fastest in the given array. This is in contrast to the default behaviour of our dg::evaluate function, which produces vectors where the first dimension of the given grid varies fastest. Thus, when defining variable dimensions the dimension name of the first grid dimension needs to come last.

Note

The unlimited dimension, if present, must be the first dimension.

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group.

Template Parameters

T	set the type of the variable

Parameters

atts	Attributes to put for the variable

std::map<std::string, dg::file::nc_att_t> atts = {{"axis", "T"}};
file.def_var_as<double>("another", {"time"}, atts);
att = file.get_att_as<std::string>( "axis", "another");
CHECK( att == "T");

defput_dim()

template<class ContainerType , class Attributes = std::map<std::string, nc_att_t>>

void dg::file::SerialNcFile::defput_dim ( std::string name, const Attributes & atts, const ContainerType & abscissas )

inline

Define a dimension and define and write to a dimension variable in one go.

Parameters

name	Name of the dimension and associated dimension variable
atts	Suggested attributes is for example "axis" : "X" which enable paraview to recognize the dimension as the x axis and "long_name" : "X-coordinate in Cartesian coordinates"
abscissas	values to write to the dimension variable (the dimension size is inferred from abscissas.size() and the type is inferred from ContainerType

Note

This function is mainly intended to define dimensions from a grid as in

    // Generate a grid
    const double x0 = 0., x1 = 2.*M_PI;
    dg::x::CartesianGrid2d grid( x0,x1,x0,x1,3,10,10
#ifdef WITH_MPI
    , comm
#endif
    );
    // and put dimensions to file
    file.defput_dim( "x", {{"axis", "X"},
        {"long_name", "x-coordinate in Cartesian system"}},
        grid.abscissas(0));
    file.defput_dim( "y", {{"axis", "Y"},
        {"long_name", "y-coordinate in Cartesian system"}},
        grid.abscissas(1));

Template Parameters

ContainerType

May be anything that dg::get_tensor_category recognises as a dg::SharedVectorTag. In defput* members the value type determines the NetCDF type of the variable to define

Note

In both put* and get* members it is possible for data to have a different value type from the defined value type of the variable that is being written/read, the NetCDF C-API simply converts it to the requested type. For example a variable declared as NC_DOUBLE can be read as/written from integer or float and vice versa.

Here is the call graph for this function:

defput_var()

template<class ContainerType , class Attributes = std::map<std::string, nc_att_t>, std::enable_if_t< dg::is_vector_v< ContainerType, SharedVectorTag >, bool > = true>

void dg::file::SerialNcFile::defput_var ( std::string name, const std::vector< std::string > & dim_names, const Attributes & atts, const NcHyperslab & slab, const ContainerType & data )

inline

Define and put a variable in one go.

Very convenient to "just write" a variable to a file:

// Generate some data and write to file
dg::x::HVec data = dg::evaluate( function, grid);
// Defne and write a variable in one go
file.defput_var( "variable", {"y", "x"},
            {{"long_name", "A long explanation"}, {"unit", "m/s"}},
            grid, data);

Short for

def_var_as<dg::get_value_type<ContainerType>>( name, dim_names, atts);
put_var( name, slab, data);

Here is the call graph for this function:

del_att()

void dg::file::SerialNcFile::del_att ( std::string att_name, std::string id = "" )

inline

Remove an attribute named att_name from variable id.

Parameters

att_name	Attribute to delete
id	Variable name in the current group or empty string, in which case the attributes refer to the current group

Note

Attributes are the only thing you can delete in a NetCDF file. You cannot delete variables or dimensions or groups

file.put_att({"same", "thing"});
CHECK( file.att_is_defined( "same"));
file.del_att( "same", "");
CHECK( not file.att_is_defined( "same"));

dim_is_defined()

bool dg::file::SerialNcFile::dim_is_defined ( std::string name ) const

inline

Check for existence of the dimension named name

// This is how to fully check a dimension
std::map<std::string, int> map {{"x",0}, {"y",1}, {"z", 2}};
for( auto str : map)
{
    CHECK( file.dim_is_defined( str.first));
    CHECK( file.var_is_defined( str.first));
    CHECK( file.get_dim_size( str.first) == grid.shape(str.second));
    auto abs = grid.abscissas(str.second) , test(abs);
    // In MPI when mode == nc_write only the group containing rank 0 in file comm reads
    file.get_var( str.first, {grid.axis(str.second)}, test);
    dg::blas1::axpby( 1.,abs,-1., test);
    double result = sqrt(dg::blas1::dot( test, test));
    CHECK( result < 1e-15);
}

get_att_as()

template<class T >

T dg::file::SerialNcFile::get_att_as ( std::string att_name, std::string id = "" ) const

inline

Get an attribute named att_name of the group or variable id.

Template Parameters

T	Any type in dg::file::nc_att_t or nc_att_t in which case the type specific nc attribute getters are called or std::vector<type> in which case the general nc_get_att is called

Parameters

att_name	Name of the attribute
id	Variable name in the current group or empty string, in which case the attribute refers to the current group

Returns

Attribute cast to type T

file.put_att( std::tuple{"half truth", NC_INT, 21});
int half = file.get_att_as<int>( "half truth");
CHECK( half == 21);

get_att_vec_as()

template<class T >

std::vector< T > dg::file::SerialNcFile::get_att_vec_as ( std::string att_name, std::string id = "" ) const

inline

Short for get_att_as<std::vector<T>>( id, att_name);

Here is the call graph for this function:

get_atts()

std::map< std::string, nc_att_t > dg::file::SerialNcFile::get_atts ( std::string id = "" ) const

inline

Short for get_atts_as<nc_att_t>( id)

std::map<std::string, dg::file::nc_att_t> att;
att["text"] = "Hello World!";
att["number"] = 3e-4;
att["int"] = -1;
att["uint"] = 10;
att["bool"] = true;
att["realarray"]  = std::vector{-1.1, 42.3};
att["realarray"]  = std::vector{-1.1, 42.3};
att["realnumber"] = -1.1;
att["intarray"]   = std::vector{-11, 423};
att["uintarray"]  = std::vector{11, 423};
att["boolarray"]  = std::vector{true, false};
file.put_atts(att);
file.close();
auto mode = GENERATE( dg::file::nc_nowrite, dg::file::nc_write);
file.open( "test.nc", mode);
auto read = file.get_atts();
CHECK( read == att);

Here is the call graph for this function:

get_atts_as()

template<class T >

std::map< std::string, T > dg::file::SerialNcFile::get_atts_as ( std::string id = "" ) const

inline

Read all NetCDF attributes of a certain type.

For example

Note

byte attributes are mapped to boolean values (0b for true, 1b for false)

Returns

A Dictionary containing all the attributes of a certain type for the variable or file. Can be empty if no attribute is present.

Parameters

id	Variable name in the current group or empty string, in which case the attributes refer to the current group

Template Parameters

T	can be a primitive type like int or double or a vector thereof std::vector<int> or a dg::file::nc_att_t in which case attributes of heterogeneous types are captured

get_current_path()

std::filesystem::path dg::file::SerialNcFile::get_current_path ( ) const

inline

Get the absolute path of the current group.

get_dim_size()

size_t dg::file::SerialNcFile::get_dim_size ( std::string name ) const

inline

Get the size of the dimension named name.

unsigned NT = file.get_dim_size( "time");
CHECK( NT == 3);

Note

The size of an unlimited dimension is the maximum of the sizes of all variables that share this dimension.

Note

There are a couple of subtle issues related to unlimited dimensions in NetCDF. First of all, all variables that share a dimension in NetCDF have the same size along that dimension. Second, it is possible to write data to any index (the start value of the hyperslab in the unlimited dimension can be anything) of a variable with an unlimited dimension. For example you can do

file.def_dim( "time", NC_UNLIMITED);
file.def_var_as<double>( "var", {"time"}, {});
file.put_var( "var", {5}, 42); // perfectly legal

The size of an unlimited dimension is the maximum of the sizes of all variables that share this dimension. If you are trying to read a variable at a slice where data was never written, the NetCDF library just fills it up with Fill Values. However, trying to read beyond the size of the unlimited dimension will fail. If a user wants to keep all unlimited variables synchronised, they unfortunately have to keep track of which variable was written themselves: See NetCDF issue

get_dims()

std::vector< std::string > dg::file::SerialNcFile::get_dims ( bool include_parents = true ) const

inline

Get all visible dimension names in the current group

The visible dimensions are all the dimensions in the current group and all its parent group.

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group.

Parameters

include_parents

per default the parent groups will be included in the search for dimensions, if false they are excluded

Returns

All visible dimension names. Hidden names do not appear.

auto dims = file.get_dims( );
std::vector<std::string> result = {"x", "y", "time"};
CHECK( dims == result);

get_dims_shape()

std::vector< size_t > dg::file::SerialNcFile::get_dims_shape ( const std::vector< std::string > & dims ) const

inline

Get the size of each dimension in dims.

Here is the call graph for this function:

get_grpid()

int dg::file::SerialNcFile::get_grpid ( ) const

inlinenoexcept

Get the NetCDF-C ID of the current group.

get_grps()

std::list< std::filesystem::path > dg::file::SerialNcFile::get_grps ( ) const

inline

Get all subgroups in the current group as absolute paths

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

get_grps_r()

std::list< std::filesystem::path > dg::file::SerialNcFile::get_grps_r ( ) const

inline

Get all subgroups recursively in the current group as absolute paths.

Think of this as ls -R

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

Note

Using the vector of paths it is possible to traverse the entire filesystem

Returns

All groups and subgroups

Attention

Does not include the current group

get_ncid()

int dg::file::SerialNcFile::get_ncid ( ) const

inlinenoexcept

Get the ncid of the underlying NetCDF C-API.

Just if for whatever reason you want to call a NetCDF C-function yourself ... just don't use it for something nasty, like closing the file or whatever

get_unlim_dims()

std::vector< std::string > dg::file::SerialNcFile::get_unlim_dims ( ) const

inline

Get all visible unlimited dimension names in the current group

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group. This function does not include the parent groups

Returns

All unlimited dimension names in current group.

get_var() [1/2]

template<class ContainerType , std::enable_if_t< dg::is_vector_v< ContainerType, SharedVectorTag >, bool > = true>

void dg::file::SerialNcFile::get_var ( std::string name, const NcHyperslab & slab, ContainerType & data ) const

inline

Read hyperslab slab from variable named name into container data.

// In MPI all ranks automatically get the right chunk of data
file.get_var( "variable", grid, data);

Parameters

name	of previously defined variable
slab	Hyperslab to read
data	Result on output (will be resized to fit hyperslab)

Template Parameters

ContainerType

May be anything that dg::get_tensor_category recognises as a dg::SharedVectorTag. In defput* members the value type determines the NetCDF type of the variable to define

Note

In both put* and get* members it is possible for data to have a different value type from the defined value type of the variable that is being written/read, the NetCDF C-API simply converts it to the requested type. For example a variable declared as NC_DOUBLE can be read as/written from integer or float and vice versa.

Here is the call graph for this function:

get_var() [2/2]

template<class T , std::enable_if_t< dg::is_scalar_v< T >, bool > = true>

void dg::file::SerialNcFile::get_var ( std::string name, const std::vector< size_t > & start, T & data ) const

inline

Read scalar from position start from variable named name.

file.get_var("time", {52}, test);
CHECK( test == 10);

Parameters

name	of previously defined variable
start	coordinate to take scalar from (can be empty for scalar variable)
data	Result on output

get_var_dims()

std::vector< std::string > dg::file::SerialNcFile::get_var_dims ( std::string name ) const

inline

Get the dimension names associated to variable name.

file.def_dim("time", NC_UNLIMITED);
file.def_dim( "y", 57);
file.def_dim( "z", 12);
file.def_var_as<double>("variable", {"time", "z","y"});
CHECK( file.get_var_dims("variable") == std::vector<std::string>{"time", "z", "y"});

Parameters

name	of the variable

Returns

list of dimension names associated with variable

Note

Dimensions are visible in a group and all of its subgroups. Now, a file can define multiple dimensions with the same name in subgroups. The dimension in the highest group will then hide the ones in the lower groups and e.g a call to def_var cannot currently use a dimension that is hidden in such a way even though the NetCDF-C API would allow to do it using the dimension ID. In contrast, get_var_dims may return dimension names that are actually hidden. It is therefore highly recommended to use unique names for all dimensions in the file. Furthermore, notice that while a dimension is visible in subgroups, its associated dimension variable is not.

Attention

Paraview seems to have issues if dimensions in a subgrup in a NetCDF file are defined before any of the dimensions in the root group.

get_var_names()

std::list< std::string > dg::file::SerialNcFile::get_var_names ( ) const

inline

Get a list of variable names in the current group.

We use std::list here because of how easy it is to sort or filter elemenets. For example

//Get a list of variables with type double
auto pred = [&file]( std::string name) {
    return file.get_var_type(name) != NC_DOUBLE;
};
file.set_grp("subgroup");
auto vars = file.get_var_names();
vars.remove_if( pred);
CHECK( vars == std::list<std::string>{"variable", "another"} );

Returns

list of variable names in current group

get_var_type()

nc_type dg::file::SerialNcFile::get_var_type ( std::string name ) const

inline

Get the NetCDF typeid of the variable named name

file.def_var_as<double>(   "variable", {"time"});
CHECK( (file.get_var_type( "variable") == NC_DOUBLE ));

grp_is_defined()

bool dg::file::SerialNcFile::grp_is_defined ( std::filesystem::path path ) const

inline

Check for existence of the group given by path.

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

Parameters

path	Absolute or relative path to the current group

Here is the call graph for this function:

is_open()

bool dg::file::SerialNcFile::is_open ( ) const

inlinenoexcept

Check if a file is associated (i.e. it is open)

dg::file::NcFile file;
file.open("test.nc", dg::file::nc_noclobber);
CHECK( file.is_open());
file.close();

open()

void dg::file::SerialNcFile::open ( const std::filesystem::path & filename, enum NcFileMode mode = nc_nowrite )

inline

Explicitly open or create a netCDF file

Parameters

filename	Name or path including the name of the netCDF file to open or create. The path may be either absolute or relative to the execution path of the program i.e. relative to std::filesystem::current_path()
mode	(see NcFileMode for nc_nowrite, nc_write, nc_clobber, nc_noclobber)

Note

Just like std::fstream opening fails if a file is already associated (is_open()) close() it before opening a new file.

dg::file::NcFile file;
file.open("test.nc", dg::file::nc_noclobber);
CHECK( file.is_open());
file.close();

operator=() [1/2]

SerialNcFile & dg::file::SerialNcFile::operator= ( const SerialNcFile & rhs )

delete

There can only be exactly one file handle per physical file.

The reason is that the destructor releases all resources and thus a copy of the file that is subsequently destroyed leaves the original in an invalid state

operator=() [2/2]

SerialNcFile & dg::file::SerialNcFile::operator= ( SerialNcFile && rhs )

default

Swap resources between two file handles.

put_att() [1/2]

void dg::file::SerialNcFile::put_att ( const std::pair< std::string, nc_att_t > & att, std::string id = "" )

inline

Put an individual attribute.

file.put_att({"title", "Hello"} );
INFO( "Silently overwrite existing attribute");
CHECK_NOTHROW( file.put_att({"title", "Hello world"}));
file.put_att({"truth", 42} );

Parameters

att	Attribute consisting of name and value
id	Variable name in the current group or empty string, in which case the attribute refers to the current group

Note

Attributes are silently overwritten. You need to manually check with att_is_defined for existence if this is a concern

put_att() [2/2]

template<class S , class T >

void dg::file::SerialNcFile::put_att ( const std::tuple< S, nc_type, T > & att, std::string id = "" )

inline

Put an individual attribute of preset type to variable id.

Template Parameters

S	std::string or const char*
T	Cannot be an nc_att_t

Parameters

att	Attribute consisting of name, type and value
id	Variable name in the current group or empty string, in which case the attribute refers to the current group

Note

Attributes are silently overwritten. You need to manually check with att_is_defined for existence if this is a concern

file.put_att( std::tuple{"half truth", NC_INT, 21});
int half = file.get_att_as<int>( "half truth");
CHECK( half == 21);

put_atts()

template<class Attributes = std::map<std::string, nc_att_t>>

void dg::file::SerialNcFile::put_atts ( const Attributes & atts, std::string id = "" )

inline

Write a collection of attributes to a NetCDF variable or file.

Example code

std::map<std::string, dg::file::nc_att_t> att;
att["text"] = "Hello World!";
att["number"] = 3e-4;
att["int"] = -1;
att["uint"] = 10;
att["bool"] = true;
att["realarray"]  = std::vector{-1.1, 42.3};
att["realarray"]  = std::vector{-1.1, 42.3};
att["realnumber"] = -1.1;
att["intarray"]   = std::vector{-11, 423};
att["uintarray"]  = std::vector{11, 423};
att["boolarray"]  = std::vector{true, false};
file.put_atts(att);
file.close();
auto mode = GENERATE( dg::file::nc_nowrite, dg::file::nc_write);
file.open( "test.nc", mode);
auto read = file.get_atts();
CHECK( read == att);

Note

boolean values are mapped to byte NetCDF attributes (0b for true, 1b for false)

Template Parameters

Attributes

Any Iterable whose values can be used in put_att i.e. either a std::pair or std::tuple

Parameters

atts	An iterable containing all the attributes for the variable or file. atts can be empty in which case no attribute is written.
id	Variable name in the current group or empty string, in which case the attributes refer to the current group

Note

Attributes are silently overwritten. You need to manually check with att_is_defined for existence if this is a concern

put_var() [1/2]

template<class ContainerType , std::enable_if_t< dg::is_vector_v< ContainerType, SharedVectorTag >, bool > = true>

void dg::file::SerialNcFile::put_var ( std::string name, const NcHyperslab & slab, const ContainerType & data )

inline

Write data to a variable.

Parameters

name	Name of the variable to write data to. Must be visible in the current group
slab	Define where the data is written. The dimension of the slab slab.ndim() must match the number of dimensions of the variable
data	to write. Size must be at least that of the slab

Template Parameters

ContainerType

May be anything that dg::get_tensor_category recognises as a dg::SharedVectorTag. In defput* members the value type determines the NetCDF type of the variable to define

Note

In both put* and get* members it is possible for data to have a different value type from the defined value type of the variable that is being written/read, the NetCDF C-API simply converts it to the requested type. For example a variable declared as NC_DOUBLE can be read as/written from integer or float and vice versa.

Note

There are a couple of subtle issues related to unlimited dimensions in NetCDF. First of all, all variables that share a dimension in NetCDF have the same size along that dimension. Second, it is possible to write data to any index (the start value of the hyperslab in the unlimited dimension can be anything) of a variable with an unlimited dimension. For example you can do

file.def_dim( "time", NC_UNLIMITED);
file.def_var_as<double>( "var", {"time"}, {});
file.put_var( "var", {5}, 42); // perfectly legal

The size of an unlimited dimension is the maximum of the sizes of all variables that share this dimension. If you are trying to read a variable at a slice where data was never written, the NetCDF library just fills it up with Fill Values. However, trying to read beyond the size of the unlimited dimension will fail. If a user wants to keep all unlimited variables synchronised, they unfortunately have to keep track of which variable was written themselves: See NetCDF issue

        typename dg::file::NcFile::Hyperslab slab{grid_out};
 
        for(unsigned i=0; i<2; i++)
        {
            INFO("Write timestep "<<i<<"\n");
            double time = i*Tmax/2.;
            file.put_var( "ptime", {i}, time);
            for( auto& record : records)
            {
                record.function ( result, grid, time);
                dg::apply( project, result, tmp);
                // Hyperslab can be constructed from hyperslab...
                file.put_var( record.name, {i, slab}, tmp);
            }
        }

Here is the call graph for this function:

put_var() [2/2]

template<class T , std::enable_if_t< dg::is_scalar_v< T >, bool > = true>

void dg::file::SerialNcFile::put_var ( std::string name, const std::vector< size_t > & start, T data )

inline

Write a single data point.

Parameters

name	Name of the variable to write data to. Must be visible in the current group
start	The coordinates (one for each dimension) to which to write data to
data	to write

Template Parameters

T	must be convertible to the datatype of the variable name

file.def_dimvar_as<double>( "time", NC_UNLIMITED, {{"axis", "T"}});
file.def_var_as<double>( "Energy", {"time"}, {{"long_name", "Energy"}});
for( unsigned i=0; i<=6; i++)
{
    file.put_var("time", {i}, i);
    if( i%2 == 0)
        file.put_var( "Energy", {i}, i);
}

Note

There are a couple of subtle issues related to unlimited dimensions in NetCDF. First of all, all variables that share a dimension in NetCDF have the same size along that dimension. Second, it is possible to write data to any index (the start value of the hyperslab in the unlimited dimension can be anything) of a variable with an unlimited dimension. For example you can do

file.def_dim( "time", NC_UNLIMITED);
file.def_var_as<double>( "var", {"time"}, {});
file.put_var( "var", {5}, 42); // perfectly legal

The size of an unlimited dimension is the maximum of the sizes of all variables that share this dimension. If you are trying to read a variable at a slice where data was never written, the NetCDF library just fills it up with Fill Values. However, trying to read beyond the size of the unlimited dimension will fail. If a user wants to keep all unlimited variables synchronised, they unfortunately have to keep track of which variable was written themselves: See NetCDF issue

rename_att()

void dg::file::SerialNcFile::rename_att ( std::string old_att_name, std::string new_att_name, std::string id = "" )

inline

Rename an attribute

file.put_att({"ttt", 42} );
CHECK( file.att_is_defined( "ttt"));
file.rename_att( "ttt", "truth");
CHECK( not file.att_is_defined( "ttt", ""));
CHECK( file.att_is_defined( "truth"));

rename_dim()

void dg::file::SerialNcFile::rename_dim ( std::string old_name, std::string new_name )

inline

Rename a dimension from old_name to new_name.

rename_grp()

void dg::file::SerialNcFile::rename_grp ( std::string old_name, std::string new_name )

inline

rename a subgroup in the current group from old_name to new_name

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

set_grp()

void dg::file::SerialNcFile::set_grp ( std::filesystem::path path = "" )

inline

Change group to path.

Groups can be thought of as directories in a NetCDF-4 file and we therefore use std::filesystem::path and use bash equivalent operations to manipulate them

        dg::file::NcFile file( "grps.nc", dg::file::nc_clobber);
        // mkdir subgroup
        file.def_grp( "subgrp");
        // cd subgroup
        file.set_grp( "subgrp");
        // all subsequent calls will write to subgrp
        file.put_att( {"title", "This is a subgrp attribute"});
        auto path = file.get_current_path();
        CHECK( path == "/subgrp");
        // mkdir -p
        file.def_grp_p("/subgrp/subgrp2/subgrp3/subgrp4");
        CHECK( file.grp_is_defined( "/subgrp/subgrp2/subgrp3"));
 
        // cd ..
        file.set_grp(".."); // go to parent group
        // cd
        file.set_grp(); // go to root group
 
        // mv subgrp sub
        file.rename_grp( "subgrp", "sub");
 
        // ls
        auto subgrps = file.get_grps();
        // ls -R
        auto allgrps = file.get_grps_r();
 
        file.close();

All subsequent calls to atts, dims and vars are made to that group

Parameters

path	can be absolute or relative to the current group. Empty string or "/" goes back to root group. "." is the current group and returns immediately.

Here is the call graph for this function:

sync()

void dg::file::SerialNcFile::sync ( )

inline

Call nc_sync.

var_is_defined()

bool dg::file::SerialNcFile::var_is_defined ( std::string name ) const

inline

Check if variable named name is defined in the current group

file.def_var_as<int>( "scalar", {}, {});
CHECK( file.var_is_defined( "scalar"));

---

The documentation for this struct was generated from the following file:

• nc_file.h