input.hpp

Go to the documentation of this file.

// -----------------------------------------------------------------------------------------------------
// Copyright (c) 2006-2021, Knut Reinert & Freie Universität Berlin
// Copyright (c) 2016-2021, Knut Reinert & MPI für molekulare Genetik
// This file may be used, modified and/or redistributed under the terms of the 3-clause BSD-License
// shipped with this file and also available at: https://github.com/seqan/seqan3/blob/master/LICENSE.md
// -----------------------------------------------------------------------------------------------------
 
#pragma once
 
#include <cassert>
#include <seqan3/std/concepts>
#include <seqan3/std/filesystem>
#include <fstream>
#include <seqan3/std/ranges>
#include <string>
#include <variant>
#include <vector>
 
#include <seqan3/alignment/decorator/gap_decorator.hpp>
#include <seqan3/alphabet/adaptation/char.hpp>
#include <seqan3/alphabet/aminoacid/aa27.hpp>
#include <seqan3/alphabet/cigar/cigar.hpp>
#include <seqan3/alphabet/nucleotide/dna15.hpp>
#include <seqan3/alphabet/nucleotide/dna5.hpp>
#include <seqan3/alphabet/quality/phred42.hpp>
#include <seqan3/alphabet/quality/qualified.hpp>
#include <seqan3/io/detail/in_file_iterator.hpp>
#include <seqan3/io/detail/misc_input.hpp>
#include <seqan3/io/detail/record.hpp>
#include <seqan3/io/exception.hpp>
#include <seqan3/io/sam_file/format_bam.hpp>
#include <seqan3/io/sam_file/format_sam.hpp>
#include <seqan3/io/sam_file/input_format_concept.hpp>
#include <seqan3/io/sam_file/record.hpp>
#include <seqan3/io/sam_file/sam_flag.hpp>
#include <seqan3/io/stream/concept.hpp>
#include <seqan3/utility/tuple/concept.hpp>
#include <seqan3/utility/type_list/traits.hpp>
#include <seqan3/utility/type_traits/detail/transformation_trait_or.hpp>
#include <seqan3/utility/views/repeat_n.hpp>
#include <seqan3/utility/views/slice.hpp>
 
namespace seqan3
{
 
// ---------------------------------------------------------------------------------------------------------------------
// sam_file_input_traits
// ---------------------------------------------------------------------------------------------------------------------
 
template <typename t>
SEQAN3_CONCEPT sam_file_input_traits = requires (t v)
{
    // field::seq
    requires writable_alphabet<typename t::sequence_alphabet>;
    requires writable_alphabet<typename t::sequence_legal_alphabet>;
    requires explicitly_convertible_to<typename t::sequence_legal_alphabet, typename t::sequence_alphabet>;
    requires sequence_container<typename t::template sequence_container<typename t::sequence_alphabet>>;
 
    // field::id
    requires sequence_container<typename t::template id_container<char>>;
 
    // field::qual
    requires writable_quality_alphabet<typename t::quality_alphabet>;
    requires sequence_container<typename t::template quality_container<typename t::quality_alphabet>>;
 
    // field::ref_seq
    // either ref_info_not_given or a range over ranges over alphabet (e.g. std::vector<dna4_vector>)
    requires std::same_as<typename t::ref_sequences, ref_info_not_given> || requires ()
    {
        requires alphabet<std::ranges::range_reference_t<std::ranges::range_reference_t<typename t::ref_sequences>>>;
    };
 
    // field::ref_id
    requires alphabet<std::ranges::range_reference_t<std::ranges::range_reference_t<typename t::ref_ids>>> &&
             (!std::same_as<typename t::ref_sequences, ref_info_not_given> ||
              writable_alphabet<std::ranges::range_reference_t<std::ranges::range_reference_t<typename t::ref_ids>>>);
    requires std::ranges::forward_range<std::ranges::range_reference_t<typename t::ref_ids>>;
    requires std::ranges::forward_range<typename t::ref_ids>;
 
    // field::offset is fixed to int32_t
    // field::ref_offset is fixed to std::optional<int32_t>
    // field::flag is fixed to seqan3::sam_flag
    // field::mapq is fixed to uint8_t
    // field::evalue is fixed to double
    // field::bitscore is fixed to double
    // field::mate is fixed to std::tuple<ref_id_container<ref_id_alphabet>, ref_offset_type, int32_t>
 
    // field::alignment
    // the alignment type cannot be configured.
    // Type of tuple entry 1 (reference) is set to
    // 1) a std::ranges::subrange over std::ranges::range_value_t<typename t::ref_sequences> if reference information was given
    // or 2) a "dummy" sequence type:
    // views::repeat_n(sequence_alphabet{}, size_t{}) | std::views::transform(detail::access_restrictor_fn{})
    // Type of tuple entry 2 (query) is set to
    // 1) a std::ranges::subrange over std::ranges::range_value_t<typename t::ref_sequences> if reference information was given
    // or 2) a "dummy" sequence type:
};
 
// ---------------------------------------------------------------------------------------------------------------------
// sam_file_input_default_traits
// ---------------------------------------------------------------------------------------------------------------------
 
template <typename ref_sequences_t = ref_info_not_given, typename ref_ids_t = std::deque<std::string>>
struct sam_file_input_default_traits
{
    using sequence_alphabet                     = dna5;
 
    using sequence_legal_alphabet               = dna15;
 
    template <typename _sequence_alphabet>
    using sequence_container                    = std::vector<_sequence_alphabet>;
 
    template <typename _id_alphabet>
    using id_container                          = std::basic_string<_id_alphabet>;
 
    using quality_alphabet                      = phred42;
 
    template <typename _quality_alphabet>
    using quality_container                     = std::vector<_quality_alphabet>;
 
    using ref_sequences                         = ref_sequences_t;
 
    using ref_ids                               = ref_ids_t;
};
 
// ---------------------------------------------------------------------------------------------------------------------
// sam_file_input
// ---------------------------------------------------------------------------------------------------------------------
 
template <
    sam_file_input_traits traits_type_ = sam_file_input_default_traits<>,
    detail::fields_specialisation selected_field_ids_ = fields<field::seq,
                                                               field::id,
                                                               field::offset,
                                                               field::ref_id,
                                                               field::ref_offset,
                                                               field::alignment,
                                                               field::cigar,
                                                               field::mapq,
                                                               field::qual,
                                                               field::flag,
                                                               field::mate,
                                                               field::tags,
                                                               field::header_ptr>,
    detail::type_list_of_sam_file_input_formats valid_formats_ = type_list<format_sam, format_bam>>
class sam_file_input
{
public:
    using traits_type           = traits_type_;
    using selected_field_ids    = selected_field_ids_;
    using valid_formats         = valid_formats_;
    using stream_char_type      = char;
 
private:
    using dummy_ref_type = decltype(views::repeat_n(typename traits_type::sequence_alphabet{}, size_t{}) |
                                    std::views::transform(detail::access_restrictor_fn{}));
 
    using ref_sequence_unsliced_type =
        detail::lazy_conditional_t<std::ranges::range<typename traits_type::ref_sequences const>,
                                  detail::lazy<std::ranges::range_reference_t,
                                               typename traits_type::ref_sequences const>,
                                  dummy_ref_type>;
 
    using ref_sequence_sliced_type = decltype(std::declval<ref_sequence_unsliced_type>() | views::slice(0, 0));
public:
    using sequence_type            = typename traits_type::template sequence_container<
                                         typename traits_type::sequence_alphabet>;
    using id_type                  = typename traits_type::template id_container<char>;
    using offset_type              = int32_t;
    using ref_sequence_type = std::conditional_t<std::same_as<typename traits_type::ref_sequences, ref_info_not_given>,
                                                 dummy_ref_type,
                                                 ref_sequence_sliced_type>;
    using ref_id_type              = std::optional<int32_t>;
    using ref_offset_type          = std::optional<int32_t>;
    using mapq_type                = uint8_t;
    using quality_type             = typename traits_type::template quality_container<
                                         typename traits_type::quality_alphabet>;
    using flag_type                = sam_flag;
    using cigar_type               = std::vector<cigar>;
    using mate_type                = std::tuple<ref_id_type, ref_offset_type, int32_t>;
    using header_type              = sam_file_header<typename traits_type::ref_ids>;
 
private:
    using alignment_query_type = std::conditional_t<
                                     selected_field_ids::contains(field::seq),
                                     gap_decorator<
                                         decltype(std::declval<sequence_type &>() | views::slice(0, 0))>,
                                     typename traits_type::template sequence_container<
                                         gapped<typename traits_type::sequence_alphabet>>>;
 
public:
    using alignment_type = std::tuple<gap_decorator<ref_sequence_type>, alignment_query_type>;
 
    using field_types = type_list<sequence_type,
                                  id_type,
                                  offset_type,
                                  ref_id_type,
                                  ref_offset_type,
                                  alignment_type,
                                  std::vector<cigar>,
                                  mapq_type,
                                  quality_type,
                                  flag_type,
                                  mate_type,
                                  sam_tag_dictionary,
                                  header_type *>;
 
    using field_ids = fields<field::seq,
                             field::id,
                             field::offset,
                             field::ref_id,
                             field::ref_offset,
                             field::alignment,
                             field::cigar,
                             field::mapq,
                             field::qual,
                             field::flag,
                             field::mate,
                             field::tags,
                             field::header_ptr>;
 
    static_assert([] () constexpr
                  {
                      for (field f : selected_field_ids::as_array)
                          if (!field_ids::contains(f))
                              return false;
                      return true;
                  }(),
                  "You selected a field that is not valid for alignment files, please refer to the documentation "
                  "of sam_file_input::field_ids for the accepted values.");
 
    using record_type = sam_record<detail::select_types_with_ids_t<field_types, field_ids, selected_field_ids>,
                                   selected_field_ids>;
 
    using value_type        = record_type;
    using reference         = record_type &;
    using const_reference   = void;
    using size_type         = size_t;
    using difference_type   = std::make_signed_t<size_t>;
    using iterator          = detail::in_file_iterator<sam_file_input>;
    using const_iterator    = void;
    using sentinel          = std::default_sentinel_t;
 
    sam_file_input() = delete;
    sam_file_input(sam_file_input const &) = delete;
    sam_file_input & operator=(sam_file_input const &) = delete;
    sam_file_input(sam_file_input &&) = default;
    sam_file_input & operator=(sam_file_input &&) = default;
    ~sam_file_input() = default;
 
    sam_file_input(std::filesystem::path filename,
                   selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new std::ifstream{}, stream_deleter_default}
    {
        init_by_filename(std::move(filename));
    }
 
    template <input_stream stream_t, sam_file_input_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_t>::char_type, stream_char_type>
    sam_file_input(stream_t & stream,
                   file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                   selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{&stream, stream_deleter_noop}
    {
        init_by_format<file_format>();
    }
 
    template <input_stream stream_t, sam_file_input_format file_format>
        requires std::same_as<typename std::remove_reference_t<stream_t>::char_type, stream_char_type>
    sam_file_input(stream_t && stream,
                   file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                   selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new stream_t{std::move(stream)}, stream_deleter_default}
    {
        init_by_format<file_format>();
    }
 
    sam_file_input(std::filesystem::path filename,
                   typename traits_type::ref_ids & ref_ids,
                   typename traits_type::ref_sequences & ref_sequences,
                   selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new std::ifstream{}, stream_deleter_default}
    {
        // initialize reference information
        set_references(ref_ids, ref_sequences);
 
        init_by_filename(std::move(filename));
    }
 
    template <input_stream stream_t, sam_file_input_format file_format>
    sam_file_input(stream_t & stream,
                   typename traits_type::ref_ids & ref_ids,
                   typename traits_type::ref_sequences & ref_sequences,
                   file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                   selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{&stream, stream_deleter_noop}
    {
        // initialize reference information
        set_references(ref_ids, ref_sequences);
 
        init_by_format<file_format>();
    }
 
    template <input_stream stream_t, sam_file_input_format file_format>
    sam_file_input(stream_t && stream,
                   typename traits_type::ref_ids & ref_ids,
                   typename traits_type::ref_sequences & ref_sequences,
                   file_format const & SEQAN3_DOXYGEN_ONLY(format_tag),
                   selected_field_ids const & SEQAN3_DOXYGEN_ONLY(fields_tag) = selected_field_ids{}) :
        primary_stream{new stream_t{std::move(stream)}, stream_deleter_default}
    {
        // initialize reference information
        set_references(ref_ids, ref_sequences);
 
        init_by_format<file_format>();
    }
 
    // explicitly delete rvalues for reference information
    sam_file_input(std::filesystem::path,
                   typename traits_type::ref_ids &&,
                   typename traits_type::ref_sequences &&,
                   selected_field_ids const &) = delete;
 
    template <input_stream stream_t, sam_file_input_format file_format>
    sam_file_input(stream_t &&,
                   typename traits_type::ref_ids &&,
                   typename traits_type::ref_sequences &&,
                   file_format const &,
                   selected_field_ids const &) = delete;
 
    iterator begin()
    {
        // buffer first record
        if (!first_record_was_read)
        {
            read_next_record();
            first_record_was_read = true;
        }
 
        return {*this};
    }
 
    sentinel end() noexcept
    {
        return {};
    }
 
    reference front() noexcept
    {
        return *begin();
    }
 
    sam_file_input_options<typename traits_type::sequence_legal_alphabet> options;
 
    header_type & header()
    {
        // make sure header is read
        if (!first_record_was_read)
        {
            read_next_record();
            first_record_was_read = true;
        }
 
        return *header_ptr;
    }
 
protected:
 
    void init_by_filename(std::filesystem::path filename)
    {
        primary_stream->rdbuf()->pubsetbuf(stream_buffer.data(), stream_buffer.size());
        static_cast<std::basic_ifstream<char> *>(primary_stream.get())->open(filename,
                                                                             std::ios_base::in | std::ios::binary);
        // open stream
        if (!primary_stream->good())
            throw file_open_error{"Could not open file " + filename.string() + " for reading."};
 
        secondary_stream = detail::make_secondary_istream(*primary_stream, filename);
        detail::set_format(format, filename);
    }
 
    template <typename format_type>
    void init_by_format()
    {
        static_assert(list_traits::contains<format_type, valid_formats>,
                      "You selected a format that is not in the valid_formats of this file.");
 
        format = detail::sam_file_input_format_exposer<format_type>{};
        secondary_stream = detail::make_secondary_istream(*primary_stream);
    }
 
    std::unique_ptr<header_type> header_ptr{new header_type{}};
 
    record_type record_buffer;
    std::vector<char> stream_buffer{std::vector<char>(1'000'000)};
 
    using stream_ptr_t = std::unique_ptr<std::basic_istream<stream_char_type>,
                                         std::function<void(std::basic_istream<stream_char_type>*)>>;
    static void stream_deleter_noop(std::basic_istream<stream_char_type> *) {}
    static void stream_deleter_default(std::basic_istream<stream_char_type> * ptr) { delete ptr; }
 
    stream_ptr_t primary_stream{nullptr, stream_deleter_noop};
    stream_ptr_t secondary_stream{nullptr, stream_deleter_noop};
 
    bool first_record_was_read{false};
    bool at_end{false};
 
    using format_type = typename detail::variant_from_tags<valid_formats,
                                                           detail::sam_file_input_format_exposer>::type;
 
    format_type format;
 
    typename traits_type::ref_sequences const * reference_sequences_ptr{nullptr};
 
    template <std::ranges::forward_range ref_sequences_t>
    void set_references(typename traits_type::ref_ids & ref_ids, ref_sequences_t && ref_sequences)
    {
        assert(std::ranges::distance(ref_ids) == std::ranges::distance(ref_sequences));
 
        header_ptr = std::unique_ptr<header_type>{std::make_unique<header_type>(ref_ids)};
        reference_sequences_ptr = &ref_sequences;
 
        // initialise reference map and ref_dict if ref_ids are non-empty
        for (int32_t idx = 0; idx < std::ranges::distance(ref_ids); ++idx)
        {
            header_ptr->ref_id_info.emplace_back(std::ranges::distance(ref_sequences[idx]), "");
 
            if constexpr (std::ranges::contiguous_range<std::ranges::range_reference_t<
                                                            typename traits_type::ref_ids>> &&
                          std::ranges::sized_range<std::ranges::range_reference_t<typename traits_type::ref_ids>> &&
                          std::ranges::borrowed_range<std::ranges::range_reference_t<typename traits_type::ref_ids>>)
            {
                auto && id = header_ptr->ref_ids()[idx];
                header_ptr->ref_dict[std::span{std::ranges::data(id), std::ranges::size(id)}] = idx;
            }
            else
            {
                header_ptr->ref_dict[header_ptr->ref_ids()[idx]] = idx;
            }
        }
    }
 
    void read_next_record()
    {
        // clear the record
        record_buffer.clear();
        detail::get_or_ignore<field::header_ptr>(record_buffer) = header_ptr.get();
 
        // at end if we could not read further
        if (std::istreambuf_iterator<stream_char_type>{*secondary_stream} ==
            std::istreambuf_iterator<stream_char_type>{})
        {
            at_end = true;
            return;
        }
 
        auto call_read_func = [this] (auto & ref_seq_info)
        {
            std::visit([&] (auto & f)
            {
                f.read_alignment_record(*secondary_stream,
                                        options,
                                        ref_seq_info,
                                        *header_ptr,
                                        detail::get_or_ignore<field::seq>(record_buffer),
                                        detail::get_or_ignore<field::qual>(record_buffer),
                                        detail::get_or_ignore<field::id>(record_buffer),
                                        detail::get_or_ignore<field::offset>(record_buffer),
                                        detail::get_or_ignore<field::ref_seq>(record_buffer),
                                        detail::get_or_ignore<field::ref_id>(record_buffer),
                                        detail::get_or_ignore<field::ref_offset>(record_buffer),
                                        detail::get_or_ignore<field::alignment>(record_buffer),
                                        detail::get_or_ignore<field::cigar>(record_buffer),
                                        detail::get_or_ignore<field::flag>(record_buffer),
                                        detail::get_or_ignore<field::mapq>(record_buffer),
                                        detail::get_or_ignore<field::mate>(record_buffer),
                                        detail::get_or_ignore<field::tags>(record_buffer),
                                        detail::get_or_ignore<field::evalue>(record_buffer),
                                        detail::get_or_ignore<field::bit_score>(record_buffer));
            }, format);
        };
 
        assert(!format.valueless_by_exception());
 
        if constexpr (!std::same_as<typename traits_type::ref_sequences, ref_info_not_given>)
            call_read_func(*reference_sequences_ptr);
        else
            call_read_func(std::ignore);
    }
 
    friend iterator;
};
 
template <input_stream stream_type, sam_file_input_format file_format, detail::fields_specialisation selected_field_ids>
sam_file_input(stream_type && stream, file_format const &, selected_field_ids const &)
    -> sam_file_input<typename sam_file_input<>::traits_type, // actually use the default
                      selected_field_ids,
                      type_list<file_format>>;
 
template <input_stream stream_type, sam_file_input_format file_format, detail::fields_specialisation selected_field_ids>
sam_file_input(stream_type & stream, file_format const &, selected_field_ids const &)
    -> sam_file_input<typename sam_file_input<>::traits_type, // actually use the default
                      selected_field_ids,
                      type_list<file_format>>;
 
template <input_stream stream_type, sam_file_input_format file_format>
sam_file_input(stream_type && stream, file_format const &)
    -> sam_file_input<typename sam_file_input<>::traits_type, // actually use the default
                      typename sam_file_input<>::selected_field_ids, // actually use the default
                      type_list<file_format>>;
 
template <input_stream stream_type, sam_file_input_format file_format>
sam_file_input(stream_type & stream, file_format const &)
    -> sam_file_input<typename sam_file_input<>::traits_type, // actually use the default
                      typename sam_file_input<>::selected_field_ids, // actually use the default
                      type_list<file_format>>;
 
template <std::ranges::forward_range ref_ids_t,
          std::ranges::forward_range ref_sequences_t,
          detail::fields_specialisation selected_field_ids>
sam_file_input(std::filesystem::path path, ref_ids_t &, ref_sequences_t &, selected_field_ids const &)
    -> sam_file_input<sam_file_input_default_traits<std::remove_reference_t<ref_sequences_t>,
                                                    std::remove_reference_t<ref_ids_t>>,
                      selected_field_ids,
                      typename sam_file_input<>::valid_formats>;  // actually use the default
 
template <std::ranges::forward_range ref_ids_t,
          std::ranges::forward_range ref_sequences_t>
sam_file_input(std::filesystem::path path, ref_ids_t &, ref_sequences_t &)
    -> sam_file_input<sam_file_input_default_traits<std::remove_reference_t<ref_sequences_t>,
                                                    std::remove_reference_t<ref_ids_t>>,
                      typename sam_file_input<>::selected_field_ids, // actually use the default
                      typename sam_file_input<>::valid_formats>;     // actually use the default
 
template <input_stream stream_type,
          std::ranges::forward_range ref_ids_t,
          std::ranges::forward_range ref_sequences_t,
          sam_file_input_format file_format,
          detail::fields_specialisation selected_field_ids>
sam_file_input(stream_type && stream, ref_ids_t &, ref_sequences_t &, file_format const &, selected_field_ids const &)
    -> sam_file_input<sam_file_input_default_traits<std::remove_reference_t<ref_sequences_t>,
                                                    std::remove_reference_t<ref_ids_t>>,
                      selected_field_ids,
                      type_list<file_format>>;
 
template <input_stream stream_type,
          std::ranges::forward_range ref_ids_t,
          std::ranges::forward_range ref_sequences_t,
          sam_file_input_format file_format,
          detail::fields_specialisation selected_field_ids>
sam_file_input(stream_type & stream, ref_ids_t &, ref_sequences_t &, file_format const &, selected_field_ids const &)
    -> sam_file_input<sam_file_input_default_traits<std::remove_reference_t<ref_sequences_t>,
                                                    std::remove_reference_t<ref_ids_t>>,
                      selected_field_ids,
                      type_list<file_format>>;
 
template <input_stream stream_type,
          std::ranges::forward_range ref_ids_t,
          std::ranges::forward_range ref_sequences_t,
          sam_file_input_format file_format>
sam_file_input(stream_type && stream, ref_ids_t &, ref_sequences_t &, file_format const &)
    -> sam_file_input<sam_file_input_default_traits<std::remove_reference_t<ref_sequences_t>,
                                                    std::remove_reference_t<ref_ids_t>>,
                      typename sam_file_input<>::selected_field_ids, // actually use the default
                      type_list<file_format>>;
 
template <input_stream stream_type,
          std::ranges::forward_range ref_ids_t,
          std::ranges::forward_range ref_sequences_t,
          sam_file_input_format file_format>
sam_file_input(stream_type & stream, ref_ids_t &, ref_sequences_t &, file_format const &)
    -> sam_file_input<sam_file_input_default_traits<std::remove_reference_t<ref_sequences_t>,
                                                    std::remove_reference_t<ref_ids_t>>,
                      typename sam_file_input<>::selected_field_ids, // actually use the default
                      type_list<file_format>>;
 
} // namespace seqan3