sequence_file/input.hpp

Go to the documentation of this file.

// SPDX-FileCopyrightText: 2006-2025 Knut Reinert & Freie Universität Berlin
// SPDX-FileCopyrightText: 2016-2025 Knut Reinert & MPI für molekulare Genetik
// SPDX-License-Identifier: BSD-3-Clause

 
#pragma once
 
#include <cassert>
#include <filesystem>
#include <fstream>
#include <string>
#include <variant>
#include <vector>
 
#include <seqan3/alphabet/adaptation/char.hpp>
#include <seqan3/alphabet/aminoacid/aa27.hpp>
#include <seqan3/alphabet/nucleotide/dna15.hpp>
#include <seqan3/alphabet/nucleotide/dna5.hpp>
#include <seqan3/alphabet/quality/phred42.hpp>
#include <seqan3/alphabet/quality/qualified.hpp>
#include <seqan3/io/detail/in_file_iterator.hpp>
#include <seqan3/io/detail/misc_input.hpp>
#include <seqan3/io/detail/record.hpp>
#include <seqan3/io/exception.hpp>
#include <seqan3/io/sam_file/format_sam.hpp>
#include <seqan3/io/sequence_file/format_embl.hpp>
#include <seqan3/io/sequence_file/format_fasta.hpp>
#include <seqan3/io/sequence_file/format_fastq.hpp>
#include <seqan3/io/sequence_file/format_genbank.hpp>
#include <seqan3/io/sequence_file/input_format_concept.hpp>
#include <seqan3/io/sequence_file/record.hpp>
#include <seqan3/io/stream/concept.hpp>
#include <seqan3/utility/type_list/traits.hpp>
 
namespace seqan3
{
 
// ----------------------------------------------------------------------------
// sequence_file_input_traits
// ----------------------------------------------------------------------------

template <typename t>
concept sequence_file_input_traits = requires (t v) {
    requires writable_alphabet<typename t::sequence_alphabet>;
    requires writable_alphabet<typename t::sequence_legal_alphabet>;
    requires detail::is_char_adaptation_v<typename t::sequence_alphabet>
                 || explicitly_convertible_to<typename t::sequence_legal_alphabet, typename t::sequence_alphabet>;
    requires sequence_container<typename t::template sequence_container<typename t::sequence_alphabet>>;
 
    requires writable_alphabet<typename t::id_alphabet>;
    requires sequence_container<typename t::template id_container<typename t::id_alphabet>>;
 
    requires writable_quality_alphabet<typename t::quality_alphabet>;
    requires sequence_container<typename t::template quality_container<typename t::quality_alphabet>>;
};
 
// ----------------------------------------------------------------------------
// sequence_file_input_default_traits
// ----------------------------------------------------------------------------

struct sequence_file_input_default_traits_dna
{

    using sequence_alphabet = dna5;

    using sequence_legal_alphabet = dna15;

    template <typename _sequence_alphabet>
    using sequence_container = std::vector<_sequence_alphabet>;

    using id_alphabet = char;

    template <typename _id_alphabet>
    using id_container = std::basic_string<_id_alphabet>;

    using quality_alphabet = phred42;

    template <typename _quality_alphabet>
    using quality_container = std::vector<_quality_alphabet>;

};

struct sequence_file_input_default_traits_aa : sequence_file_input_default_traits_dna
{

    using sequence_alphabet = aa27;

    using sequence_legal_alphabet = aa27;
};
 
// ----------------------------------------------------------------------------
// sequence_file_input
// ----------------------------------------------------------------------------

 
template <sequence_file_input_traits traits_type_ = sequence_file_input_default_traits_dna,
          detail::fields_specialisation selected_field_ids_ = fields<field::seq, field::id, field::qual>,
          detail::type_list_of_sequence_file_input_formats valid_formats_ =
              type_list<format_embl, format_fasta, format_fastq, format_genbank, format_sam>>
class sequence_file_input
{
public:
    using traits_type = traits_type_;
    using selected_field_ids = selected_field_ids_;
    using valid_formats = valid_formats_;
    using stream_char_type = char;

    using field_ids = fields<field::seq, field::id, field::qual>;
 
    static_assert(
        []() constexpr
        {
            for (field f : selected_field_ids::as_array)
                if (!field_ids::contains(f))
                    return false;
            return true;
        }(),
        "You selected a field that is not valid for sequence files, please refer to the documentation "
        "of sequence_file_input::field_ids for the accepted values.");

    using sequence_type = typename traits_type::template sequence_container<typename traits_type::sequence_alphabet>;
    using id_type = typename traits_type::template id_container<typename traits_type::id_alphabet>;
    using quality_type = typename traits_type::template quality_container<typename traits_type::quality_alphabet>;
    using field_types = type_list<sequence_type, id_type, quality_type>;

    using record_type = sequence_record<detail::select_types_with_ids_t<field_types, field_ids, selected_field_ids>,
                                        selected_field_ids>;

    using value_type = record_type;
    using reference = record_type &;
    using const_reference = void;
    using size_type = size_t;
    using difference_type = std::make_signed_t<size_t>;
    using iterator = detail::in_file_iterator<sequence_file_input>;
    using const_iterator = void;
    using sentinel = std::default_sentinel_t;

    sequence_file_input() = delete;
    sequence_file_input(sequence_file_input const &) = delete;
    sequence_file_input & operator=(sequence_file_input const &) = delete;
    sequence_file_input(sequence_file_input &&) = default;
    sequence_file_input & operator=(sequence_file_input &&) = default;
    ~sequence_file_input() = default;

    sequence_file_input(std::filesystem::path filename,
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new std::ifstream{}, stream_deleter_default}
    {
        primary_stream->rdbuf()->pubsetbuf(stream_buffer.data(), stream_buffer.size());
        static_cast<std::basic_ifstream<char> *>(primary_stream.get())
            ->open(filename, std::ios_base::in | std::ios::binary);
 
        if (!primary_stream->good())
            throw file_open_error{"Could not open file " + filename.string() + " for reading."};
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream, filename);
 
        // initialise format handler or throw if format is not found
        using format_variant_t =
            typename detail::variant_from_tags<valid_formats, detail::sequence_file_input_format_exposer>::type;
        format_variant_t format_variant{};
        detail::set_format(format_variant, filename);
 
        std::visit(
            [&](auto && selected_format)
            {
                using format_t = std::remove_cvref_t<decltype(selected_format)>;
                format = std::make_unique<selected_sequence_format<format_t>>();
            },
            format_variant);
    }
    /* NOTE(h-2): Curiously we do not need a user-defined deduction guide for the above constructor.
     * A combination of default template parameters and auto-deduction guides works as expected,
     * independent of whether the second/optional parameter is specified or not, i.e. it is possible
     * to auto-deduct and overwrite a single template parameter out of the four if the optional parameter
     * is specified and use the default otherwise.
     */

    template <input_stream stream_t, sequence_file_input_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_t>::char_type, stream_char_type>
    sequence_file_input(stream_t & stream,
                        file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{&stream, stream_deleter_noop},
        format{std::make_unique<selected_sequence_format<file_format>>()}
    {
        static_assert(list_traits::contains<file_format, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream);
    }

    template <input_stream stream_t, sequence_file_input_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_t>::char_type, stream_char_type>
    sequence_file_input(stream_t && stream,
                        file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                        selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new stream_t{std::move(stream)}, stream_deleter_default},
        format{std::make_unique<selected_sequence_format<file_format>>()}
    {
        static_assert(list_traits::contains<file_format, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
 
        // possibly add intermediate compression stream
        secondary_stream = detail::make_secondary_istream(*primary_stream);
    }


    iterator begin()
    {
        // buffer first record
        if (!first_record_was_read)
        {
            read_next_record();
            first_record_was_read = true;
        }
 
        return {*this};
    }

    sentinel end() noexcept
    {
        return {};
    }

    reference front() noexcept
    {
        return *begin();
    }


    using sequence_file_input_options_type = sequence_file_input_options<typename traits_type::sequence_legal_alphabet>;
    sequence_file_input_options_type options;
 
protected:

    record_type record_buffer;
    std::vector<char> stream_buffer{std::vector<char>(1'000'000)};
    std::streampos position_buffer{};

    using stream_ptr_t = std::unique_ptr<std::basic_istream<stream_char_type>,
                                         std::function<void(std::basic_istream<stream_char_type> *)>>;
    static void stream_deleter_noop(std::basic_istream<stream_char_type> *)
    {}
    static void stream_deleter_default(std::basic_istream<stream_char_type> * ptr)
    {
        delete ptr;
    }

    stream_ptr_t primary_stream{nullptr, stream_deleter_noop};
    stream_ptr_t secondary_stream{nullptr, stream_deleter_noop};

    bool first_record_was_read{false};
    bool at_end{false};
 
private:
    void read_next_record()
    {
        // clear the record
        record_buffer.clear();
 
        // at end if we could not read further
        if ((std::istreambuf_iterator<stream_char_type>{*secondary_stream}
             == std::istreambuf_iterator<stream_char_type>{}))
        {
            at_end = true;
            return;
        }
 
        format->read_sequence_record(*secondary_stream, record_buffer, position_buffer, options);
    }

    struct sequence_format_base
    {
        sequence_format_base() = default;                                         
        sequence_format_base(sequence_format_base const &) = default;             
        sequence_format_base(sequence_format_base &&) = default;                  
        sequence_format_base & operator=(sequence_format_base const &) = default; 
        sequence_format_base & operator=(sequence_format_base &&) = default;      
        virtual ~sequence_format_base() = default;                                

        virtual void read_sequence_record(std::istream & instream,
                                          record_type & record_buffer,
                                          std::streampos & position_buffer,
                                          sequence_file_input_options_type const & options) = 0;
    };

    template <typename format_t>
    struct selected_sequence_format final : public sequence_format_base
    {
        selected_sequence_format() = default;                                             
        selected_sequence_format(selected_sequence_format const &) = default;             
        selected_sequence_format(selected_sequence_format &&) = default;                  
        selected_sequence_format & operator=(selected_sequence_format const &) = default; 
        selected_sequence_format & operator=(selected_sequence_format &&) = default;      
        ~selected_sequence_format() = default;                                            

        void read_sequence_record(std::istream & instream,
                                  record_type & record_buffer,
                                  std::streampos & position_buffer,
                                  sequence_file_input_options_type const & options) override
        {
            // read new record
            {
                _format.read_sequence_record(instream,
                                             options,
                                             position_buffer,
                                             detail::get_or_ignore<field::seq>(record_buffer),
                                             detail::get_or_ignore<field::id>(record_buffer),
                                             detail::get_or_ignore<field::qual>(record_buffer));
            }
        }

        detail::sequence_file_input_format_exposer<format_t> _format{};
    };

    std::unique_ptr<sequence_format_base> format{};

    friend iterator;
};


template <input_stream stream_type, sequence_file_input_format file_format>
sequence_file_input(stream_type & stream,
                    file_format const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,        // actually use the default
                           typename sequence_file_input<>::selected_field_ids, // default field ids.
                           type_list<file_format>>;

template <input_stream stream_type, sequence_file_input_format file_format>
sequence_file_input(stream_type && stream,
                    file_format const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type,        // actually use the default
                           typename sequence_file_input<>::selected_field_ids, // default field ids.
                           type_list<file_format>>;

template <input_stream stream_type,
          sequence_file_input_format file_format,
          detail::fields_specialisation selected_field_ids>
sequence_file_input(stream_type && stream,
                    file_format const &,
                    selected_field_ids const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type, // actually use the default
                           selected_field_ids,
                           type_list<file_format>>;

template <input_stream stream_type,
          sequence_file_input_format file_format,
          detail::fields_specialisation selected_field_ids>
sequence_file_input(stream_type & stream,
                    file_format const &,
                    selected_field_ids const &)
    -> sequence_file_input<typename sequence_file_input<>::traits_type, // actually use the default
                           selected_field_ids,
                           type_list<file_format>>;
 
} // namespace seqan3