PTreeFile.cpp

Go to the documentation of this file.

/*
 * Copyright 2011-2016 Universiteit Antwerpen
 *
 * Licensed under the EUPL, Version 1.1 or  as soon they will be approved by
 * the European Commission - subsequent versions of the EUPL (the "Licence");
 * You may not use this work except in compliance with the Licence.
 * You may obtain a copy of the Licence at: http://ec.europa.eu/idabc/eupl5
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the Licence is distributed on an "AS IS" basis,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Licence for the specific language governing
 * permissions and limitations under the Licence.
 */
#include "PTreeFile.h"

#include "util/misc/Exception.h"
#include "util/misc/log_debug.h"
#include "util/misc/XmlWriterSettings.h"

#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>

#include <algorithm>
#include <exception>
#include <iostream>
#include <fstream>
#include <string>

using namespace std;
using namespace boost::property_tree;
using namespace boost::property_tree::xml_parser;
using namespace SimPT_Sim::Util;

// ------------------------------------------------------------------------------
// Implementations independent of boost iostreams / zlib conditional
// ------------------------------------------------------------------------------
namespace SimPT_Sim {
namespace Util {

bool PTreeFile::IsGzipped(const string& path)
{
        string s = path;
        transform(s.begin(), s.end(), s.begin(), ::tolower);
        return s.rfind(".gz") != string::npos;
}

bool PTreeFile::IsPTreeFile(const string& path)
{
        string s = path;
        transform(s.begin(), s.end(), s.begin(), ::tolower);
        return path.rfind(".xml.gz") != string::npos || path.rfind("xml") != string::npos ;
}

string PTreeFile::GetCompleteSuffix(const string& path)
{
        // pos1 points to position past last "/" (if present) or start of string
        const auto posa = path.rfind("/");
        const auto pos1 = (posa != string::npos) ? posa + 1 : 0;

        // [pos1, pos1 + len1] defines substr up to first "."
        const auto posb = path.substr(pos1).find(".");
        const auto len1 = (posb != string::npos) ? path.length() - posb : 0;

        return path.substr(pos1 + posb + 1, len1);
}

string PTreeFile::GetBaseNameExcludingLabel(const string& path)
{
        // pos1 points to position past last "/" (if present) or start of string
        const auto posa = path.rfind("/");
        const auto pos1 = (posa != string::npos) ? posa + 1 : 0;

        // [pos1, pos1 + len1] defines substr up to first "."
        const auto posb = path.substr(pos1).find(".");
        const auto len1 = (posb != string::npos) ? posb : path.length() - pos1;

        // [pos1, len2] defines the substr up to last "_"
        const auto posc = path.substr(pos1, len1).rfind("_");
        const auto len2 = (posc != string::npos) ? posc : len1;

        return path.substr(pos1, len2);
}

string PTreeFile::GetBaseNameIncludingLabel(const string& path)
{
        // pos1 points to position past last "/" (if present) or start of string
        const auto posa = path.rfind("/");
        const auto pos1 = (posa != string::npos) ? posa + 1 : 0;

        // [pos1, pos1 + len1] defines substr up to first "."
        const auto posb = path.substr(pos1).find(".");
        const auto len1 = (posb != string::npos) ? posb : path.length() - pos1;

        return path.substr(pos1, len1);
}

string PTreeFile::GetLabel(const string& path)
{
        // pos1 points to position past last "/" (if present) or start of string
        const auto posa = path.rfind("/");
        const auto pos1 = (posa != string::npos) ? posa + 1 : 0;

        // [pos1, pos1 + len1] defines substr up to first "."
        const auto posb = path.substr(pos1).find(".");
        const auto len1 = (posb != string::npos) ? posb : path.length() - pos1;

        // [pos1, len2] defines the substr up to last "_"
        const auto posc = path.substr(pos1, len1).rfind("_");
        const auto len2 = (posc != string::npos) ? len1 - posc - 1 : 0;

        return path.substr(pos1 + posc + 1, len2);
}

void PTreeFile::Read(const string& path, ptree& pt)
{
        if (IsGzipped(path)) {
                ReadXmlGz(path, pt);
        } else {
                ReadXml(path, pt);
        }
}

void PTreeFile::ReadXml(const string& path, ptree& pt)
{
        read_xml(path, pt, trim_whitespace);
}

void PTreeFile::Write(const string& path, const ptree& pt)
{
        if (IsGzipped(path)) {
                WriteXmlGz(path, pt);
        } else {
                WriteXml(path, pt);
        }
}

void PTreeFile::WriteXml(const string& path, const ptree& pt)
{
        write_xml(path, pt, locale(), XmlWriterSettings::GetTab());
}

} // namespace Util
} // namespace SimPT_Sim


#ifdef USE_BOOST_GZIP

        // ------------------------------------------------------------------------------
        // Implementation of ReadXmlGz with Boost iostreams
        // ------------------------------------------------------------------------------
        #include <boost/iostreams/copy.hpp>
        #include <boost/iostreams/device/file.hpp>
        #include <boost/iostreams/device/file_descriptor.hpp>
        #include <boost/iostreams/filter/gzip.hpp>
        #include <boost/iostreams/filtering_stream.hpp>
        #include <boost/iostreams/filtering_streambuf.hpp>
        #include <boost/iostreams/traits.hpp>

        namespace SimPT_Sim {
        namespace Util {

        using namespace boost::iostreams;

        void PTreeFile::Compress(const string& in_path, const string& out_path)
        {
                // buffered compression with boost iostreams
                file_source in(in_path);
                filtering_ostream out;
                out.push(gzip_compressor());
                out.push(file_sink(out_path));
                copy(in, out);
        }

        void PTreeFile::Decompress(const string& in_path, const string& out_path)
        {
                // buffered decompression with boost iostreams
                filtering_istream in;
                in.push(gzip_decompressor());
                in.push(file_source(in_path));
                file_sink out(out_path);
                copy(in, out);
        }

        constexpr bool PTreeFile::IsBoostGzipAvailable()
        {
                return true;
        }

        void PTreeFile::ReadXmlGz(const string& path, ptree& pt)
        {
                filtering_streambuf<input> in_buffer;
                in_buffer.push(gzip_decompressor());
                in_buffer.push(file_source(path));
                filtering_istream ptree_in_stream(in_buffer);
                read_xml(ptree_in_stream, pt, trim_whitespace);
        }

        void PTreeFile::WriteXmlGz(const string& path, const ptree& pt)
        {
                filtering_streambuf<output> out_buffer;
                out_buffer.push(gzip_compressor());
                out_buffer.push(file_sink(path));
                filtering_ostream ptree_out_stream(out_buffer);
                write_xml(ptree_out_stream, pt, XmlWriterSettings::GetTab());
        }

        } // namespace Util
        } // namespace SimPT_Sim

#else

        // ------------------------------------------------------------------------------
        // Implementation of ReadXmlGz with zlib
        // ------------------------------------------------------------------------------
        #include <array>
        #include "zlib.h"

        namespace SimPT_Sim {
        namespace Util {

        void PTreeFile::Compress(const string& in_path, const string& out_path)
        {
                // Move file into memory, parse as ptree, then write as compressed xml
                ptree pt;
                ReadXml(in_path, pt);
                WriteXmlGz(out_path, pt);
        }

        void PTreeFile::Decompress(const string& in_path, const string& out_path)
        {
                // Move file into memory while decompressing, parse as ptree, then write as xml
                ptree pt;
                ReadXmlGz(in_path, pt);
                WriteXml(out_path, pt);
        }

        constexpr bool PTreeFile::IsBoostGzipAvailable()
        {
                return false;
        }

        void PTreeFile::ReadXmlGz(const string& path, ptree& pt)
        {
                // Reads chunks of a file, decompresses them and add them to a string that
                // represents the uncompressed file. That string is then parsed as boost ptree.

                ifstream infile;
                infile.open(path, ifstream::in | ifstream::binary);
                stringstream ptree_in_stream;

                if (infile) {
                        const unsigned int in_buffer_size  = 1 << 15;
                        const unsigned int out_buffer_size = 1 << 18; // larger because we are decompressing

                        // Allocate zlib input/output buffers on heap so Java doesn't
                        // complain about any stack overflows
                        auto in_buffer  = unique_ptr<array<char, in_buffer_size>>(
                                new array<char, in_buffer_size>());
                        auto out_buffer = unique_ptr<array<char, out_buffer_size>>(
                                new array<char, out_buffer_size>());

                        z_stream s;
                        s.zalloc    = Z_NULL;
                        s.zfree     = Z_NULL;
                        s.opaque    = Z_NULL;
                        s.avail_in  = 0;
                        s.next_in   = Z_NULL;

                        if (inflateInit2(&s, 31) != Z_OK) {
                                infile.close();
                                throw runtime_error("could not initialize zlib inflate stream.");
                        }

                        int status = 0;
                        do {
                                try {
                                        infile.read(in_buffer->data(), in_buffer_size); // may throw
                                } catch (exception& e) {
                                        inflateEnd(&s);
                                        infile.close();
                                        throw;
                                }
                                s.avail_in = infile.gcount();
                                if (s.avail_in == 0)
                                        break; // EOF
                                s.next_in = (unsigned char*) in_buffer->data();

                                do {
                                        s.avail_out = out_buffer_size;
                                        s.next_out = (unsigned char*) out_buffer->data();

                                        status = inflate(&s, Z_NO_FLUSH);
                                        if (status == Z_NEED_DICT || status == Z_DATA_ERROR || status == Z_MEM_ERROR) {
                                                inflateEnd(&s);
                                                infile.close();
                                                throw runtime_error("inflate error.");
                                        }
                                        unsigned int have = out_buffer_size - s.avail_out;

                                        // accumulate output buffer contents in string
                                        ptree_in_stream.write(out_buffer->data(), have);
                                        if (ptree_in_stream.fail()) {
                                                inflateEnd(&s);
                                                infile.close();
                                                throw runtime_error("stringstream write error.");
                                        }
                                } while (s.avail_out == 0);

                        } while (status != Z_STREAM_END);

                        inflateEnd(&s);
                        infile.close();

                        if (status != Z_STREAM_END) {
                                throw runtime_error("data error.");
                        }

                        ptree_in_stream.seekg(0); // "rewind" stream

                } else {
                        throw runtime_error("could not open file.");
                }

                read_xml(ptree_in_stream, pt, trim_whitespace);
        }

        void PTreeFile::WriteXmlGz(const string& path, const ptree& pt)
        {
                stringstream ptree_out_stream;
                write_xml(ptree_out_stream, pt, XmlWriterSettings::GetTab());

                const unsigned int in_buffer_size = 1 << 18;
                const unsigned int out_buffer_size = 1 << 15; // smaller because we are compressing
                ofstream outfile(path, ofstream::out | ofstream::binary | ofstream::trunc);
                if (!outfile) {
                        throw runtime_error("could not open file for writing.");
                }
                unsigned char in_buffer[in_buffer_size];
                unsigned char out_buffer[out_buffer_size];

                z_stream s;
                s.zalloc    = Z_NULL;
                s.zfree     = Z_NULL;
                s.opaque    = Z_NULL;
                s.avail_in  = 0;

                if (deflateInit2(&s, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15+16, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
                        outfile.close();
                        throw runtime_error("could not initialize zlib deflate stream.");
                }

                int flush;
                do {
                        try {
                                ptree_out_stream.read((char*) in_buffer, in_buffer_size); // may throw
                        } catch (exception& e) {
                                deflateEnd(&s);
                                outfile.close();
                                throw;
                        }
                        s.avail_in = ptree_out_stream.gcount();
                        s.next_in = in_buffer;
                        flush = ptree_out_stream.eof() ? Z_FINISH : Z_NO_FLUSH;
                        do {
                                s.avail_out = out_buffer_size;
                                s.next_out  = out_buffer;
                                deflate(&s, flush);
                                int have = out_buffer_size - s.avail_out;
                                outfile.write((char*) out_buffer, have);
                                if (outfile.fail()) {
                                        deflateEnd(&s);
                                        outfile.close();
                                        throw runtime_error("outfile write error.");
                                }
                        } while (s.avail_out == 0);
                        // done when last data in file processed
                } while (flush != Z_FINISH);

                // clean up and return
                deflateEnd(&s);
                outfile.close();
        }

        } // namespace Util
        } // namespace SimPT_Sim

#endif