train_base.py

# German base static model for sentence comparisons, RAG & classifications.
# Inspired in January 25 by Tom Aarsens: "Train 400x faster Static Embedding Models with Sentence Transformers"
# check: https://huggingface.co/blog/static-embeddings#code
# and check: https://sbert.net/docs/sentence_transformer/training_overview.html
# for training parameters, check also: https://huggingface.co/docs/transformers/en/main_classes/trainer
# First test build since May, 25th as I found the time.
# The datasets are mainly based upon german and english european table dataset training snippets
# Main idea is to use only open licensed material that can also be used commercially.
#
# This is experimental minimal EN & mainly DE only.
#
# With local prepared texts building the train/test-split takes about 3 minutes.
# Training on a GTX-2070 SUPER 8GB (with prepared training material) needs ~2h.

from timeit import default_timer as timer
import gc
import os
import random
import logging
import datasets
from datasets import load_dataset, Dataset, DatasetDict, concatenate_datasets
from sentence_transformers import (
    SentenceTransformer,
    SentenceTransformerTrainer,
    SentenceTransformerTrainingArguments,
    SentenceTransformerModelCardData,
    SimilarityFunction,
)
from sentence_transformers.losses import MatryoshkaLoss, MultipleNegativesRankingLoss
from sentence_transformers.training_args import BatchSamplers, MultiDatasetBatchSamplers
from sentence_transformers.models.StaticEmbedding import StaticEmbedding
from sentence_transformers.util import paraphrase_mining
from sentence_transformers.evaluation import NanoBEIREvaluator

from transformers import AutoTokenizer # sadly no blingfire

# as Sentence Transformers uses PyTorch AND TensorFlow - I need to tune it for my system
import tensorflow as tf
import torch

## Model Version
version = '1'
sts_basename = 'sts-mrl-en-de-base'

## MULTILINGUAL bert-base (original): ~414MB model
#tokenizer_model = 'google-bert/bert-base-multilingual-uncased'
### follwing are some different tokenizers to play around with - all of them were tested and only 'dbmdz/bert-base-german-uncased' is more effective for the german language by only a size of 243MB.
## GERMAN ONLY: ~243MB model
tokenizer_model = 'dbmdz/bert-base-german-uncased'
## GERMAN ONLY: ~122MB model
#tokenizer_model = 'deepset/gelectra-base'
## GERMAN ONLY; ~243MB model
#tokenizer_model = 'deepset/gbert-base'
## MULTILINGUAL roBERTa: ~977MB model
#tokenizer_model = 'FacebookAI/xlm-roberta-base'
## ModernBert: ~197MB model – as a test for v0.05a
#tokenizer_model = 'answerdotai/ModernBERT-base'

logging.basicConfig(
    format="%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO
)
random.seed(12)

def load_train_eval_datasets():
    """
    Either load the train and eval datasets from disk or load them from the datasets library & save them to disk.

    Upon saving to disk, we quit() to ensure that the datasets are not loaded into memory before training.

    The order of sets here is not the same as later on in the full training/eval-sets!!!
    """
    try:
        train_dataset = DatasetDict.load_from_disk("base_datasets/train_dataset")
        eval_dataset = DatasetDict.load_from_disk("base_datasets/eval_dataset")
        return train_dataset, eval_dataset
    except FileNotFoundError:
        print("No prepared dataset found. Building ...")
#
        # Build the datasets.
        # we do the biggest thing in the beginning
        print("Loading mMARCO-distilled-de-hn dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/mmarco-de-distilled-scored
        # original: https://huggingface.co/datasets/unicamp-dl/mmarco
        # git: https://github.com/unicamp-dl/mMARCO
        # License: Apache-2.0
        # distilled & filtered: 254660
        # Original set without Hard Negatives unused
        #mmarco_de_scored = load_dataset('MarcGrumpyOlejak/mmarco-de-distilled-scored', split="train").filter(lambda _: _['score_sts'] >= 0.26)
        #mmarco_de_scored = mmarco_de_scored.select_columns(['query', 'positive', 'negative'])
        #mmarco_de_scored = mmarco_de_scored.train_test_split(test_size=10000, seed=12)
        #mmarco_de_scored_train_ds: Dataset = mmarco_de_scored["train"]
        #mmarco_de_scored_eval_ds: Dataset = mmarco_de_scored["test"]
        #
        # filtered, split as/with hard negatives and remaining sentences
        mmarco_de_3hn_ds = load_dataset('parquet', data_files={'mmarco-de-distilled_3hn/3_hard_negatives/*.parquet'}, split="train")
        mmarco_de_3hn_ds = mmarco_de_3hn_ds.train_test_split(test_size=0.02, seed=12)
        mmarco_de_3hn_train_dataset: Dataset = mmarco_de_3hn_ds["train"]
        mmarco_de_3hn_eval_dataset: Dataset = mmarco_de_3hn_ds["test"]
        #
        mmarco_de_2hn_ds = load_dataset('parquet', data_files={'mmarco-de-distilled_3hn/2_hard_negatives/*.parquet'}, split="train")
        mmarco_de_2hn_ds = mmarco_de_2hn_ds.train_test_split(test_size=0.02, seed=12)
        mmarco_de_2hn_train_dataset: Dataset = mmarco_de_2hn_ds["train"]
        mmarco_de_2hn_eval_dataset: Dataset = mmarco_de_2hn_ds["test"]
        #
        mmarco_de_1hn_ds = load_dataset('parquet', data_files={'mmarco-de-distilled_3hn/1_hard_negatives/*.parquet'}, split="train")
        mmarco_de_1hn_ds = mmarco_de_1hn_ds.train_test_split(test_size=0.02, seed=12)
        mmarco_de_1hn_train_dataset: Dataset = mmarco_de_1hn_ds["train"]
        mmarco_de_1hn_eval_dataset: Dataset = mmarco_de_1hn_ds["test"]
        #
        mmarco_de_0hn_ds = load_dataset('parquet', data_files={'mmarco-de-distilled_3hn/0_hard_negatives/*.parquet'}, split="train")
        mmarco_de_0hn_ds = mmarco_de_0hn_ds.train_test_split(test_size=0.02, seed=12)
        mmarco_de_0hn_train_dataset: Dataset = mmarco_de_0hn_ds["train"]
        mmarco_de_0hn_eval_dataset: Dataset = mmarco_de_0hn_ds["test"]
        print("Loaded mMARCO-distilled-de-hn dataset.")
#
        print("Loading local prepared wikipedia-22-12-de datasets...")
        # (need to upload the local version to build it)
        # check: load_dataset('deutsche-telekom/wikipedia-22-12-de-dpr')
        # License: MIT
        # Copyright (c) 2023-2024 Philip May, Deutsche Telekom AG
        # Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License by reviewing the file [LICENSE](https://github.com/telekom/mltb2/blob/main/LICENSE) in the repository.
        # version without hard negatives not loaded
        # reversed!!! deactivate hard negatives!
        name_local = 'wikipedia-22-12-de-scored'
        wp_2212_de_ds = DatasetDict.load_from_disk(f'{name_local}/{name_local}.hf')
        wp_2212_de_train_dataset: Dataset = wp_2212_de_ds["train"].select_columns(['question', 'context'])
        wp_2212_de_eval_dataset: Dataset = wp_2212_de_ds["test"].select_columns(['question', 'context'])
        #
        # instead load the hard negative version
        #name_local = 'wikipedia-22-12-de_hn'
        #wp_2212_de_train_dataset: Dataset = load_dataset('parquet', data_files={f'{name_local}/3_hard_negatives/train-*.parquet'}, split="train")
        #wp_2212_de_eval_dataset: Dataset = load_dataset('parquet', data_files={f'{name_local}/3_hard_negatives/test-*.parquet'}, split="train")
        #wp_2212_de_0_train_dataset: Dataset = load_dataset('parquet', data_files={f'{name_local}/0_hard_negatives/train-*.parquet'}, split="train")
        #wp_2212_de_0_eval_dataset: Dataset = load_dataset('parquet', data_files={f'{name_local}/0_hard_negatives/test-*.parquet'}, split="train")

        print("Loaded prepared full wikipedia-22-12-de dataset...")
#
        print("Loading swim-ir-monolingual-de-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/swim-ir-monolingual-de-scored
        # original: https://huggingface.co/datasets/nthakur/swim-ir-monolingual
        # entries: ~447000
        # filtered: 356552
        # combined: 713104
        # License: CC-BY-SA-4.0
        # Original set without Hard Negatives unsed
        #swim_ir_de_ds = load_dataset("MarcGrumpyOlejak/swim-ir-monolingual-de-scored", split="train").filter(lambda _: _['score_sts'] >= 0.26 and _['score_sts'] < 0.99 and _['query'] != '')
        #swim_ir_de_key_ds = swim_ir_de_ds.select_columns(['text', 'title'])
        #swim_ir_de_key_ds = swim_ir_de_key_ds.rename_columns({'text': 'sentence1', 'title': 'sentence2'})
        #swim_ir_de_ds = swim_ir_de_ds.select_columns(['query', 'text'])
        #swim_ir_de_ds = swim_ir_de_ds.rename_columns({'query': 'sentence1', 'text': 'sentence2'})
        #swim_ir_de_ds = concatenate_datasets([swim_ir_de_ds, swim_ir_de_key_ds])
        #swim_ir_de_ds = swim_ir_de_ds.train_test_split(test_size=10000, seed=12)
        #swim_ir_de_train_dataset: Dataset = swim_ir_de_ds["train"]
        #swim_ir_de_eval_dataset: Dataset = swim_ir_de_ds["test"]
        #
        # filtered, split and with hard negatives and remaining sentences
        swim_ir_de_ds = load_dataset('parquet', data_files={'swim-ir-monolingual-de_3hn/0_hard_negatives/*.parquet'}, split="train")
        swim_ir_de_ds = swim_ir_de_ds.train_test_split(test_size=0.02, seed=12)
        swim_ir_de_train_dataset: Dataset = swim_ir_de_ds["train"]
        swim_ir_de_eval_dataset: Dataset = swim_ir_de_ds["test"]
        swim_ir_de_3hn_ds = load_dataset('parquet', data_files={'swim-ir-monolingual-de_3hn/3_hard_negatives/*.parquet'}, split="train")
        swim_ir_de_3hn_ds = swim_ir_de_3hn_ds.train_test_split(test_size=0.02, seed=12)
        swim_ir_de_3hn_train_dataset: Dataset = swim_ir_de_3hn_ds["train"]
        swim_ir_de_3hn_eval_dataset: Dataset = swim_ir_de_3hn_ds["test"]
        #
        swim_ir_de_title_ds = load_dataset('parquet', data_files={'swim-ir-monolingual-titles-de_3hn/0_hard_negatives/*.parquet'}, split="train")
        swim_ir_de_title_3hn_ds = load_dataset('parquet', data_files={'swim-ir-monolingual-titles-de_3hn/3_hard_negatives/*.parquet'}, split="train")
        swim_ir_de_title_ds = swim_ir_de_title_ds.train_test_split(test_size=0.02, seed=12)
        swim_ir_de_title_3hn_ds = swim_ir_de_title_3hn_ds.train_test_split(test_size=0.02, seed=12)
        swim_ir_de_title_train_dataset: Dataset = swim_ir_de_title_ds['train']
        swim_ir_de_title_eval_dataset: Dataset = swim_ir_de_title_ds["test"]
        swim_ir_de_title_3hn_train_dataset: Dataset = swim_ir_de_title_3hn_ds['train']
        swim_ir_de_title_3hn_eval_dataset: Dataset = swim_ir_de_title_3hn_ds['test']
        print("Loaded swim-ir-monolingual-de-scored dataset.")
#
        print("Loading avemio_triples dataset...")
        # source: https://huggingface.co/datasets/avemio/German-RAG-EMBEDDING-TRIPLES-HESSIAN-AI
        # entries: 294234
        # License: Apache-2.0
        avemio_triples_dataset = load_dataset("avemio/German-RAG-EMBEDDING-TRIPLES-HESSIAN-AI", split="train")
        avemio_triples_dataset_dict = avemio_triples_dataset.train_test_split(test_size=10000, seed=12)
        avemio_triples_train_dataset: Dataset = avemio_triples_dataset_dict["train"]
        avemio_triples_eval_dataset: Dataset = avemio_triples_dataset_dict["test"]
        print("Loaded avemio_triples dataset.")
#
        print("Loading avemio_pairs-hn dataset...")
        # source: https://huggingface.co/datasets/avemio/German-RAG-EMBEDDING-PAIRS-HESSIAN-AI
        # entries: 1036940
        # License: Apache-2.0
        # Original dataset unused
        #avemio_pairs_dataset = load_dataset("avemio/German-RAG-EMBEDDING-PAIRS-HESSIAN-AI", split="train")
        #avemio_pairs_dataset_dict = avemio_pairs_dataset.train_test_split(test_size=10000, seed=12)
        #avemio_pairs_train_dataset: Dataset = avemio_pairs_dataset_dict["train"]
        #avemio_pairs_eval_dataset: Dataset = avemio_pairs_dataset_dict["test"]
        #
        # filtered, split and with hard negatives and remaining sentences
        avemio_pairs_3hn_ds = load_dataset('parquet', data_files={'German-RAG-EMBEDDING-PAIRS-HESSIAN-AI-3hn-350_3hn/3_hard_negatives/*.parquet', 'German-RAG-EMBEDDING-PAIRS-HESSIAN-AI-3hn-600_3hn/3_hard_negatives/*.parquet', 'German-RAG-EMBEDDING-PAIRS-HESSIAN-AI-3hn-600plus_3hn/3_hard_negatives/*.parquet',}, split="train")
        avemio_pairs_3hn_ds = avemio_pairs_3hn_ds.train_test_split(test_size=10000, seed=12)
        avemio_pairs_3hn_train_ds: Dataset = avemio_pairs_3hn_ds["train"]
        avemio_pairs_3hn_eval_ds: Dataset = avemio_pairs_3hn_ds["test"]
        del avemio_pairs_3hn_ds
        #
        avemio_pairs_0hn_ds = load_dataset('parquet', data_files={'German-RAG-EMBEDDING-PAIRS-HESSIAN-AI-3hn-350_3hn/0_hard_negatives/*.parquet', 'German-RAG-EMBEDDING-PAIRS-HESSIAN-AI-3hn-600_3hn/0_hard_negatives/*.parquet', 'German-RAG-EMBEDDING-PAIRS-HESSIAN-AI-3hn-600plus_3hn/0_hard_negatives/*.parquet',}, split="train")
        avemio_pairs_0hn_ds = avemio_pairs_0hn_ds.train_test_split(test_size=10000, seed=12)
        avemio_pairs_0hn_train_ds: Dataset = avemio_pairs_0hn_ds["train"]
        avemio_pairs_0hn_eval_ds: Dataset = avemio_pairs_0hn_ds["test"]
        del avemio_pairs_0hn_ds
        print("Loaded avemio_pairs-hn dataset.")
#
        print("Loading nq_german-hn dataset...")
        # source: https://huggingface.co/datasets/oliverguhr/natural-questions-german
        # entries: 100231
        # original source: https://ai.google.com/research/NaturalQuestions
        # License: cc-by-sa-3.0
        # without hard negatives but unused
        #nq_german_dataset = load_dataset("oliverguhr/natural-questions-german", split="train").select_columns(['query_de', 'answer_de'])
        #nq_german_dataset_dict = nq_german_dataset.train_test_split(test_size=0.02, seed=12)
        #nq_german_train_dataset: Dataset = nq_german_dataset_dict["train"]
        #nq_german_eval_dataset: Dataset = nq_german_dataset_dict["test"]
        #
        # filtered, split and with hard negatives and remaining sentences
        nq_german_en_de_a_3hn_ds = load_dataset('parquet', data_files={'natural-questions-german-en_de-a-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        nq_german_en_de_a_3hn_ds = nq_german_en_de_a_3hn_ds.train_test_split(test_size=0.02, seed=12)
        nq_german_en_de_a_3hn_train_ds: Dataset = nq_german_en_de_a_3hn_ds['train']
        nq_german_en_de_a_3hn_eval_ds: Dataset = nq_german_en_de_a_3hn_ds['test']
        #
        nq_german_en_de_3hn_ds = load_dataset('parquet', data_files={'natural-questions-german-en_de-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        nq_german_en_de_3hn_ds = nq_german_en_de_3hn_ds.train_test_split(test_size=0.02, seed=12)
        nq_german_en_de_3hn_train_ds: Dataset = nq_german_en_de_3hn_ds['train']
        nq_german_en_de_3hn_eval_ds: Dataset = nq_german_en_de_3hn_ds['test']
        #
        nq_german_3hn_ds = load_dataset('parquet', data_files={'natural-questions-german-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        nq_german_3hn_ds = nq_german_3hn_ds.train_test_split(test_size=0.02, seed=12)
        nq_german_3hn_train_ds: Dataset = nq_german_3hn_ds['train']
        nq_german_3hn_eval_ds: Dataset = nq_german_3hn_ds['test']
        #
        nq_german_1hn_ds = load_dataset('parquet', data_files={'natural-questions-german-sts_3hn/1_hard_negatives/*.parquet'}, split="train")
        nq_german_1hn_ds = nq_german_1hn_ds.train_test_split(test_size=0.02, seed=12)
        nq_german_1hn_train_ds: Dataset = nq_german_1hn_ds['train']
        nq_german_1hn_eval_ds: Dataset = nq_german_1hn_ds['test']
        print("Loaded nq_german-hn dataset.")
#
        print("Loading german-oasst1-qa-format-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/german-oasst1-qa-format-scored
        # original: https://huggingface.co/datasets/AgentWaller/german-oasst1-qa-format
        # entries: ~9800
        # License: apache-2.0
        #german_oasst1 = load_dataset("MarcGrumpyOlejak/german-oasst1-qa-format-scored").filter(lambda _: _['score_sts'] >= 0.16 and _['score_sts'] < 0.99)
        #german_oasst1_train_dataset: Dataset = german_oasst1["train"].select_columns(['input', 'output'])
        #german_oasst1_eval_dataset: Dataset = german_oasst1['validation'].select_columns(['input', 'output'])
        #
        name_local = 'german-oasst1-qa-format-hn'
        german_oasst1_hn_train_dataset: Dataset = load_dataset('parquet', data_files={f'{name_local}/3_hard_negatives/train-*.parquet'}, split="train")
        german_oasst1_hn_eval_dataset: Dataset = load_dataset('parquet', data_files={f'{name_local}/3_hard_negatives/test-*.parquet'}, split="train")
        print("Loaded german-oasst1-qa-format-scored dataset.")
#
        print("Loading germanrag-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/germanrag-scored
        # german original: https://huggingface.co/datasets/DiscoResearch/germanrag
        # original: https://huggingface.co/datasets/deepset/germandpr
        # entries: ~3300
        # filtered & modified: 4556
        # License: cc-by-4.0
        # Hint: one could 'refilter' the 'contexts' down to the selected 'answer' in 'positive_ctx_idx' and use the other answers as hard negatives.
        def list_to_string(_):
            _['contexts'] = ' '.join(_['contexts'])
            return _
        germanrag_short = load_dataset("MarcGrumpyOlejak/germanrag-scored", split='train').filter(lambda _: _['score_sts'] >= 0.16 and _['score_sts'] < 0.98 and _['positive_ctx_idx'] != -1)
        germanrag_context = germanrag_short.select_columns(['answer', 'contexts'])
        germanrag_context = germanrag_context.map(list_to_string)
        germanrag_context = germanrag_context.rename_columns({'answer': 'sentence1', 'contexts': 'sentence2'})
        germanrag_short = germanrag_short.select_columns(['question', 'answer'])
        germanrag_short = germanrag_short.rename_columns({'question': 'sentence1', 'answer': 'sentence2'})
        germanrag_short = concatenate_datasets([germanrag_short, germanrag_context])
        germanrag_short = germanrag_short.train_test_split(test_size=0.02, seed=12)
        germanrag_short_train_dataset: Dataset = germanrag_short["train"]
        germanrag_short_eval_dataset: Dataset = germanrag_short["test"]
        print("Loaded germanrag dataset.")
#
        print("Loading slimorca_dedup_german_experimental-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/slimorca_dedup_german_experimental-scored
        # german original: https://huggingface.co/datasets/jphme/slimorca_dedup_german_experimental
        # original: https://huggingface.co/datasets/Open-Orca/SlimOrca-Dedup
        # entries: ~322000
        # filtered: 305406
        # License: MIT
        # Original set without Hard Negatives unused
        #slimorca_dedup_german = load_dataset("MarcGrumpyOlejak/slimorca_dedup_german_experimental-scored").filter(lambda _: _['score_sts'] >= 0.16 and _['score_sts'] < 0.98)
        #slimorca_dedup_german = slimorca_dedup_german.select_columns(['instruction', 'response'])
        #slimorca_dedup_german = slimorca_dedup_german['train'].train_test_split(test_size=0.02, seed=12)
        #slimorca_dedup_german_train_dataset: Dataset = slimorca_dedup_german["train"]
        #slimorca_dedup_german_eval_dataset: Dataset = slimorca_dedup_german["test"]
        #
        # FILTERED, SPLIT AND WITH HARD NEGATIVES
        slimorca_dedup_3hn_ds = load_dataset('parquet', data_files={'slimorca_dedup_german_experimental-sts-negatives_3hn/3_hard_negatives/*.parquet'}, split="train")
        slimorca_dedup_3hn_ds = slimorca_dedup_3hn_ds.train_test_split(test_size=0.02, seed=12)
        slimorca_dedup_3hn_train_ds: Dataset = slimorca_dedup_3hn_ds['train']
        slimorca_dedup_3hn_eval_ds: Dataset = slimorca_dedup_3hn_ds['test']
        #
        slimorca_dedup_2hn_ds = load_dataset('parquet', data_files={'slimorca_dedup_german_experimental-sts-negatives_3hn/2_hard_negatives/*.parquet'}, split="train")
        slimorca_dedup_2hn_ds = slimorca_dedup_2hn_ds.train_test_split(test_size=0.02, seed=12)
        slimorca_dedup_2hn_train_ds: Dataset = slimorca_dedup_2hn_ds['train']
        slimorca_dedup_2hn_eval_ds: Dataset = slimorca_dedup_2hn_ds['test']
        #
        slimorca_dedup_1hn_ds = load_dataset('parquet', data_files={'slimorca_dedup_german_experimental-sts-negatives_3hn/1_hard_negatives/*.parquet'}, split="train")
        slimorca_dedup_1hn_ds = slimorca_dedup_1hn_ds.train_test_split(test_size=0.02, seed=12)
        slimorca_dedup_1hn_train_ds: Dataset = slimorca_dedup_1hn_ds['train']
        slimorca_dedup_1hn_eval_ds: Dataset = slimorca_dedup_1hn_ds['test']
        #
        slimorca_dedup_0hn_ds = load_dataset('parquet', data_files={'slimorca_dedup_german_experimental-sts-negatives_3hn/0_hard_negatives/*.parquet'}, split="train")
        slimorca_dedup_0hn_ds = slimorca_dedup_0hn_ds.train_test_split(test_size=0.02, seed=12)
        slimorca_dedup_0hn_train_ds: Dataset = slimorca_dedup_0hn_ds['train']
        slimorca_dedup_0hn_eval_ds: Dataset = slimorca_dedup_0hn_ds['test']
        print("Loaded slimorca_dedup_german_experimental-scored dataset.")
#
        print("Loading gpt-4-self-instruct-german-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/gpt-4-self-instruct-german-scored
        # original: https://huggingface.co/datasets/CausalLM/GPT-4-Self-Instruct-German
        # entries: ~10000
        # filtered: 9776
        # License: CC-BY-4.0
        #german_gpt4 = load_dataset("MarcGrumpyOlejak/gpt-4-self-instruct-german-scored").filter(lambda _: _['score_sts'] >= 0.16 and _['score_sts'] < 0.98).select_columns(['instruction', 'output'])
        #german_gpt4 = german_gpt4['train'].train_test_split(test_size=0.02, seed=12)
        #german_gpt4_train_dataset: Dataset = german_gpt4["train"]
        #german_gpt4_eval_dataset: Dataset = german_gpt4["test"]
        #
        name_local = 'gpt-4-self-instruct-german-hn'
        german_gpt4 = load_dataset('parquet', data_files={f'{name_local}/3_hard_negatives/train-*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        german_gpt4_3hn_train_dataset: Dataset = german_gpt4["train"]
        german_gpt4_3hn_eval_dataset: Dataset = german_gpt4["test"]
        print("Loaded GPT-4-Self-Instruct-German dataset.")
#
        print("Loading ultradistil-intel-orca-dpo-de-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/ultradistil-intel-orca-dpo-de-scored
        # original: https://huggingface.co/datasets/aari1995/ultradistil-intel-orca-dpo-de
        # entries: ~6000
        # filtered: ~5547
        # License: apache-2.0
        german_orca_dpo_ds = load_dataset("MarcGrumpyOlejak/ultradistil-intel-orca-dpo-de-scored").filter(lambda _: _['score_sts'] >= 0.16 and _['score_sts'] < 0.98)
        german_orca_dpo_ds = german_orca_dpo_ds.select_columns(['input', 'chosen', 'rejected'])
        german_orca_dpo_ds = german_orca_dpo_ds['train'].train_test_split(test_size=0.02, seed=12)
        german_orca_dpo_train_dataset: Dataset = german_orca_dpo_ds["train"]
        german_orca_dpo_eval_dataset: Dataset = german_orca_dpo_ds["test"]
        print("Loaded ultradistil-intel-orca-dpo-de-scored dataset.")
#
        #scored version of alpaca-gpt4_de-scored
        print("Loading alpaca-gpt4_de-scored dataset...")
        # source: https://huggingface.co/datasets/MarcGrumpyOlejak/alpaca-gpt4_de-scored
        # german original: https://huggingface.co/datasets/mayflowergmbh/alpaca-gpt4_de
        # original: https://huggingface.co/datasets/FreedomIntelligence/alpaca-gpt4-deutsch
        # entries: ~50000
        # filtered ~44845
        # License: apache-2.0
        # Original unused
        #alpaca_gpt4_de_ds = load_dataset("MarcGrumpyOlejak/alpaca-gpt4_de-scored").filter(lambda _: _['score_sts'] >= 0.16 and _['score_sts'] < 0.94)
        #alpaca_gpt4_de_ds = alpaca_gpt4_de_ds.select_columns(['instruction', 'output'])
        #alpaca_gpt4_de_ds = alpaca_gpt4_de_ds['train'].train_test_split(test_size=0.02, seed=12)
        #alpaca_gpt4_de_train_dataset: Dataset = alpaca_gpt4_de_ds["train"]
        #alpaca_gpt4_de_eval_dataset: Dataset = alpaca_gpt4_de_ds["test"]
        # filtered and hard negatives
        alpaca_gpt4_de_3hn_ds = load_dataset('parquet', data_files={'alpaca-gpt4_de_3hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        alpaca_gpt4_de_3hn_train_dataset: Dataset = alpaca_gpt4_de_3hn_ds['train']
        alpaca_gpt4_de_3hn_eval_dataset: Dataset = alpaca_gpt4_de_3hn_ds['test']
        alpaca_gpt4_de_0hn_ds = load_dataset('parquet', data_files={'alpaca-gpt4_de_3hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        alpaca_gpt4_de_0hn_train_dataset: Dataset = alpaca_gpt4_de_0hn_ds['train']
        alpaca_gpt4_de_0hn_eval_dataset: Dataset = alpaca_gpt4_de_0hn_ds['test']
        print("Loaded alpaca-gpt4_de dataset.")
#
        print("Loading DOLLY-15k (en-de) dataset...")
        # source: https://huggingface.co/datasets/argilla/databricks-dolly-15k-curated-multilingual
        # entries: ~15000
        # License: cc-by-sa-3.0
        # Original combined merged dataset unsused
        #db_dolly = load_dataset("argilla/databricks-dolly-15k-curated-multilingual", split="de")
        #db_dolly_en_de_inststruction = db_dolly.select_columns(['instruction_original_en', 'instruction']).filter(lambda _: _['instruction_original_en'] != "" and _['instruction'] != '')
        #db_dolly_en_de_inststruction = db_dolly_en_de_inststruction.rename_columns({'instruction_original_en': 'sentence1', 'instruction': 'sentence2'})
        #db_dolly_en_de_context = db_dolly.select_columns(['context_original_en', 'context']).filter(lambda _: _['context_original_en'] != "" and _['context'] != '')
        #db_dolly_en_de_context = db_dolly_en_de_context.rename_columns({'context_original_en': 'sentence1', 'context': 'sentence2'})
        #db_dolly_en_de_response = db_dolly.select_columns(['response_original_en', 'response']).filter(lambda _: _['response_original_en'] != "" and _['response'] != '')
        #db_dolly_en_de_response = db_dolly_en_de_response.rename_columns({'response_original_en': 'sentence1', 'response': 'sentence2'})
        #db_dolly_qa_de = db_dolly.select_columns(['instruction', 'response']).filter(lambda _: _['instruction'] != "" and _['response'] != '')
        #db_dolly_qa_de = db_dolly_qa_de.rename_columns({'instruction': 'sentence1', 'response': 'sentence2'})
        #db_dolly_qcontext_de = db_dolly.select_columns(['response', 'context']).filter(lambda _: _['response'] != "" and _['context'] != '')
        #db_dolly_qcontext_de = db_dolly_qcontext_de.rename_columns({'response': 'sentence1', 'context': 'sentence2'})
        #db_dolly_contextq_de = db_dolly.select_columns(['context', 'instruction']).filter(lambda _: _['context'] != "" and _['instruction'] != '')
        #db_dolly_contextq_de = db_dolly_contextq_de.rename_columns({'context': 'sentence1', 'instruction': 'sentence2'})
        # concat all small tables
        #db_dolly = concatenate_datasets([db_dolly_en_de_inststruction, db_dolly_en_de_context, db_dolly_en_de_response, db_dolly_qa_de, db_dolly_qcontext_de, db_dolly_contextq_de])
        #db_dolly_ds = db_dolly.train_test_split(test_size=0.02, seed=12)
        #db_dolly_train_dataset: Dataset = db_dolly_ds["train"]
        #db_dolly_eval_dataset: Dataset = db_dolly_ds["test"]
        #
        # hard negative versions and remaining sentences
        dolly_context_de_3hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/context-de-hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_context_de_3hn_train_ds: Dataset = dolly_context_de_3hn_ds['train']
        dolly_context_de_3hn_eval_ds: Dataset = dolly_context_de_3hn_ds['test']
        dolly_context_de_0hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/context-de-hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_context_de_0hn_train_ds: Dataset = dolly_context_de_0hn_ds['train']
        dolly_context_de_0hn_eval_ds: Dataset = dolly_context_de_0hn_ds['test']
        dolly_context_ende_3hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/context-en_de-hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_context_ende_3hn_train_ds: Dataset = dolly_context_ende_3hn_ds['train']
        dolly_context_ende_3hn_eval_ds: Dataset = dolly_context_ende_3hn_ds['test']
        # the next set is empty :D
        #dolly_context_ende_0hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/context-en_de-hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_instructions_de_3hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/instructions-de-hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_instructions_de_3hn_train_ds: Dataset = dolly_instructions_de_3hn_ds['train']
        dolly_instructions_de_3hn_eval_ds: Dataset = dolly_instructions_de_3hn_ds['test']
        dolly_instructions_de_0hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/instructions-de-hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_instructions_de_0hn_train_ds: Dataset = dolly_instructions_de_0hn_ds['train']
        dolly_instructions_de_0hn_eval_ds: Dataset = dolly_instructions_de_0hn_ds['test']
        dolly_instructions_ende_3hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/instructions-en_de-hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_instructions_ende_3hn_train_ds: Dataset = dolly_instructions_ende_3hn_ds['train']
        dolly_instructions_ende_3hn_eval_ds: Dataset = dolly_instructions_ende_3hn_ds['test']
        dolly_instructions_ende_0hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/instructions-en_de-hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_instructions_ende_0hn_train_ds: Dataset = dolly_instructions_ende_0hn_ds['train']
        dolly_instructions_ende_0hn_eval_ds: Dataset = dolly_instructions_ende_0hn_ds['test']
        dolly_responses_de_3hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/response-de-hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_responses_de_3hn_train_ds: Dataset = dolly_responses_de_3hn_ds['train']
        dolly_responses_de_3hn_eval_ds: Dataset = dolly_responses_de_3hn_ds['test']
        dolly_responses_de_0hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/response-de-hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_responses_de_0hn_train_ds: Dataset = dolly_responses_de_0hn_ds['train']
        dolly_responses_de_0hn_eval_ds: Dataset = dolly_responses_de_0hn_ds['test']
        dolly_responses_ende_3hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/response-en_de-hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_responses_ende_3hn_train_ds: Dataset = dolly_responses_ende_3hn_ds['train']
        dolly_responses_ende_3hn_eval_ds: Dataset = dolly_responses_ende_3hn_ds['test']
        dolly_responses_ende_0hn_ds = load_dataset('parquet', data_files={'databricks-dolly-15k-curated-de/response-en_de-hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        dolly_responses_ende_0hn_train_ds: Dataset = dolly_responses_ende_0hn_ds['train']
        dolly_responses_ende_0hn_eval_ds: Dataset = dolly_responses_ende_0hn_ds['test']
        print("Loaded DOLLY-15k (en-de) dataset.")
#
        print("Loading 'saf-legal_domain_german' dataset...")
        # source: https://huggingface.co/datasets/Short-Answer-Feedback/saf_legal_domain_german
        # License: CC-BY-4.0
        # entries: ~1600
        # filtered: ~1100 (score >= 0.75) and recombined
        saf_legal_de_train = load_dataset("Short-Answer-Feedback/saf_legal_domain_german", split="train").filter(lambda _: _['score'] >= 0.75)
        saf_legal_de_qa_train = saf_legal_de_train.select_columns(['question', 'provided_answer']).rename_columns({'question': 'sentence1', 'provided_answer': 'sentence2'})
        saf_legal_de_a_train = saf_legal_de_train.select_columns(['provided_answer', 'reference_answer']).rename_columns({'provided_answer': 'sentence1', 'reference_answer': 'sentence2'})
        saf_legal_de_train_ds: Dataset = concatenate_datasets([saf_legal_de_qa_train, saf_legal_de_a_train])
        # Loading & Preparing validation set
        saf_legal_de_eval = load_dataset("Short-Answer-Feedback/saf_legal_domain_german", split="validation").filter(lambda _: _['score'] >= 0.75)
        saf_legal_de_qa_eval = saf_legal_de_eval.select_columns(['question', 'provided_answer']).rename_columns({'question': 'sentence1', 'provided_answer': 'sentence2'})
        saf_legal_de_a_eval = saf_legal_de_eval.select_columns(['provided_answer', 'reference_answer']).rename_columns({'provided_answer': 'sentence1', 'reference_answer': 'sentence2'})
        saf_legal_de_eval_ds: Dataset = concatenate_datasets([saf_legal_de_qa_eval, saf_legal_de_a_eval])
        print("Loaded 'saf-legal_domain_german' dataset.")
#
        print("Loading GLS dataset...")
        # German Legal Sentences (GLS)
        # source: https://huggingface.co/datasets/lavis-nlp/german_legal_sentences
        # https://lavis-nlp.github.io/german_legal_sentences/
        # uses "custom code": https://huggingface.co/datasets/lavis-nlp/german_legal_sentences/blob/main/german_legal_sentences.py
        # License: MIT - see https://github.com/lavis-nlp/GerDaLIR
        # Original License: https://github.com/openlegaldata/oldp#MIT-1-ov-file
        # interesting fields: query.text, related.text
        # entries: 1404271
        #
        # Original unused
        #gls_pairs_dataset_dict = load_dataset("lavis-nlp/german_legal_sentences", "pairs").select_columns(['query.text', 'related.text'])
        #gls_pairs_train_dataset: Dataset = gls_pairs_dataset_dict["train"]
        #gls_pairs_eval_dataset: Dataset = gls_pairs_dataset_dict["validation"]
        #
        # Distilled and hard mined negatives
        gls_3hn = load_dataset('parquet', data_files={'german_legal_sentences_dist_3hn/3_hard_negatives/*.parquet'})['train'].train_test_split(test_size=0.02, seed=12)
        gls_3hn_train_dataset: Dataset = gls_3hn['train']
        gls_3hn_eval_dataset: Dataset = gls_3hn['test']
        gls_2hn = load_dataset('parquet', data_files={'german_legal_sentences_dist_3hn/2_hard_negatives/*.parquet'})['train'].train_test_split(test_size=0.02, seed=12)
        gls_2hn_train_dataset: Dataset = gls_2hn['train']
        gls_2hn_eval_dataset: Dataset = gls_2hn['test']
        gls_1hn = load_dataset('parquet', data_files={'german_legal_sentences_dist_3hn/1_hard_negatives/*.parquet'})['train'].train_test_split(test_size=0.02, seed=12)
        gls_1hn_train_dataset: Dataset = gls_1hn['train']
        gls_1hn_eval_dataset: Dataset = gls_1hn['test']
        gls_0hn = load_dataset('parquet', data_files={'german_legal_sentences_dist_3hn/0_hard_negatives/*.parquet'})['train'].train_test_split(test_size=0.02, seed=12)
        gls_0hn_train_dataset: Dataset = gls_0hn['train']
        gls_0hn_eval_dataset: Dataset = gls_0hn['test']
        print("Loaded GLS dataset.")
#
        print("Loading europarl EN-DE dataset...")
        # source: https://huggingface.co/datasets/sentence-transformers/parallel-sentences-europarl
        # original: https://opus.nlpl.eu/Europarl/corpus/version/Europarl
        # Info: https://opus.nlpl.eu/legacy/LREC2012.txt
        # entries: ~1.9m
        #europarl_dataset = load_dataset("sentence-transformers/parallel-sentences-europarl", "en-de", split="train")
        #europarl_dataset_dict = europarl_dataset.train_test_split(test_size=10000, seed=12)
        #europarl_train_dataset: Dataset = europarl_dataset_dict["train"]
        #europarl_eval_dataset: Dataset = europarl_dataset_dict["test"]
        #
        # filtered and 3 hard negatives and 0 negatives
        europarl_dataset_3hn = load_dataset('parquet', data_files={'parallel-sentences-europarl-redux_3hn/3_hard_negatives/*.parquet'})['train'].train_test_split(test_size=10000, seed=12)
        europarl_3hn_train_dataset: Dataset = europarl_dataset_3hn["train"]
        europarl_3hn_eval_dataset: Dataset = europarl_dataset_3hn["test"]
        #
        europarl_dataset_0hn = load_dataset('parquet', data_files={'parallel-sentences-europarl-redux_3hn/0_hard_negatives/*.parquet'})['train'].train_test_split(test_size=0.02, seed=12)
        europarl_0hn_train_dataset: Dataset = europarl_dataset_0hn["train"]
        europarl_0hn_eval_dataset: Dataset = europarl_dataset_0hn["test"]
        print("Loaded europarl EN-DE dataset.")
#
        print("Loading tatoeba EN-DE dataset...")
        # source: https://huggingface.co/datasets/sentence-transformers/parallel-sentences-tatoeba
        # original: https://tatoeba.org/
        # entries: ~330k
        #tatoeba_dataset = load_dataset("sentence-transformers/parallel-sentences-tatoeba", "en-de", split="train")
        #tatoeba_dataset_dict = tatoeba_dataset.train_test_split(test_size=10000, seed=12)
        #tatoeba_train_dataset: Dataset = tatoeba_dataset_dict["train"]
        #tatoeba_eval_dataset: Dataset = tatoeba_dataset_dict["test"]
        #
        tatoeba_dataset_3hn = load_dataset('parquet', data_files={'parallel-sentences-tatoeba-en-de-hn/3_hard_negatives/*.parquet'})['train'].train_test_split(test_size=10000, seed=12)
        tatoeba_3hn_train_dataset: Dataset = tatoeba_dataset_3hn["train"]
        tatoeba_3hn_eval_dataset: Dataset = tatoeba_dataset_3hn["test"]
        #
        tatoeba_dataset_0hn = load_dataset('parquet', data_files={'parallel-sentences-tatoeba-en-de-hn/0_hard_negatives/*.parquet'})['train'].train_test_split(test_size=0.02, seed=12)
        tatoeba_0hn_train_dataset: Dataset = tatoeba_dataset_0hn["train"]
        tatoeba_0hn_eval_dataset: Dataset = tatoeba_dataset_0hn["test"]
        print("Loaded tatoeba EN-DE dataset.")
#
        print("Loading WikiMatrix EN-DE dataset...")
        # source: (EN-DE) https://huggingface.co/datasets/sentence-transformers/parallel-sentences-wikimatrix
        # License: CC BY-SA 4.0
        # entries: ~344k
        # Original dataset not used
        #wikimatrix_dataset = load_dataset("sentence-transformers/parallel-sentences-wikimatrix", "en-de", split="train")
        #wikimatrix_dataset_dict = wikimatrix_dataset.train_test_split(test_size=10000, seed=12)
        #wikimatrix_train_dataset: Dataset = wikimatrix_dataset_dict["train"]
        #wikimatrix_eval_dataset: Dataset = wikimatrix_dataset_dict["test"]
        #
        # scored and filtered hard negative version and remaining sentences
        wikimatrix_3hn_ds = load_dataset('parquet', data_files={'parallel-sentences-wikimatrix-hn_3hn/3_hard_negatives/train-*.parquet'}, split='train')
        wikimatrix_3hn_ds = wikimatrix_3hn_ds.train_test_split(test_size=10000, seed=12)
        wikimatrix_3hn_train_ds: Dataset = wikimatrix_3hn_ds["train"]
        wikimatrix_3hn_eval_ds: Dataset = wikimatrix_3hn_ds["test"]
        #
        wikimatrix_0hn_ds = load_dataset('parquet', data_files={'parallel-sentences-wikimatrix-hn_3hn/0_hard_negatives/train-*.parquet'}, split='train')
        wikimatrix_0hn_ds = wikimatrix_0hn_ds.train_test_split(test_size=0.02, seed=12)
        wikimatrix_0hn_train_ds: Dataset = wikimatrix_0hn_ds["train"]
        wikimatrix_0hn_eval_ds: Dataset = wikimatrix_0hn_ds["test"]
        #
        print("Loaded WikiMatrix EN-DE dataset.")
#
        print("Loading Wikipedia-Abstract DE dataset...")
        # source: https://huggingface.co/datasets/laion/Wikipedia-Abstract
        # License: MIT
        # entries: 2.57M
        # comment: relicensing a Wikipedia text to MIT is a bit unusual as it was Creative Commons Attribution-ShareAlike 4.0 and/or GNU Free Documentation License
        # original version unused
        #wikipedia_abstract_ds = load_dataset("laion/Wikipedia-Abstract", "German", split="train").select_columns(['Title', 'Abstract'])
        #wikipedia_abstract_ds = wikipedia_abstract_ds.train_test_split(test_size=10000, seed=12)
        #wikipedia_abstract_train_dataset: Dataset = wikipedia_abstract_ds["train"]
        #wikipedia_abstract_eval_dataset: Dataset = wikipedia_abstract_ds["test"]
        #
        # hard negative version and remaining sentences
        wikipedia_abstract_3hn_ds = load_dataset('parquet', data_files={'Wikipedia-Abstract-distilled_3hn/3_hard_negatives/train-*.parquet'}, split='train')
        wikipedia_abstract_3hn_ds = wikipedia_abstract_3hn_ds.train_test_split(test_size=10000, seed=12)
        wikipedia_abstract_3hn_train_dataset: Dataset = wikipedia_abstract_3hn_ds["train"]
        wikipedia_abstract_3hn_eval_dataset: Dataset = wikipedia_abstract_3hn_ds["test"]
        #
        wikipedia_abstract_0hn_ds = load_dataset('parquet', data_files={'Wikipedia-Abstract-distilled_3hn/0_hard_negatives/train-*.parquet'}, split='train')
        wikipedia_abstract_0hn_ds = wikipedia_abstract_0hn_ds.train_test_split(test_size=0.02, seed=12)
        wikipedia_abstract_0hn_train_dataset: Dataset = wikipedia_abstract_0hn_ds["train"]
        wikipedia_abstract_0hn_eval_dataset: Dataset = wikipedia_abstract_0hn_ds["test"]
        print("Loaded Wikipedia-Abstract DE dataset.")
#
        print("Loading wiktionary GDG-D DE dataset...")
        # source: https://huggingface.co/jfeil/GermanDefinitionGeneration-Distillation
        # License: gpl-3.0
        # entries: ~900k
        #
        # GermanDefinitionGeneration-Distillation_3hn
        wiktionary_gdg_de_3hn_train_ds: Dataset = load_dataset('parquet', data_files={'GermanDefinitionGeneration-Distillation_3hn/3_hard_negatives/train-*.parquet'}, split='train')
        wiktionary_gdg_de_3hn_eval_ds: Dataset = load_dataset('parquet', data_files={'GermanDefinitionGeneration-Distillation_3hn/3_hard_negatives/validation-*.parquet'}, split='train')
        #
        # still needs optimisation
        wiktionary_gdg_de_short_ds = load_dataset("jfeil/GermanDefinitionGeneration-Distillation")
        wiktionary_gdg_de_short_ds = wiktionary_gdg_de_short_ds.select_columns(['context_sentence', 'title'])
        wiktionary_gdg_de_short_train_dataset: Dataset = wiktionary_gdg_de_short_ds["train"]
        wiktionary_gdg_de_short_eval_dataset: Dataset = wiktionary_gdg_de_short_ds["test"]
        print("Loaded GDG-D DE dataset.")
#
        print("Loading wmt24pp dataset...")
        # source: https://huggingface.co/datasets/google/wmt24pp
        # License: Apache-2.0
        # interesting fields: source, target
        # entries: 960 (after filtering of 'is_bad_source')
        wmt24pp_dataset = load_dataset("google/wmt24pp", "en-de_DE", split="train").filter(lambda _: _["is_bad_source"] == False)
        wmt24pp_dataset = wmt24pp_dataset.select_columns(['source', 'target'])
        wmt24pp_dataset_dict = wmt24pp_dataset.train_test_split(test_size=0.02, seed=12)
        wmt24pp_train_dataset: Dataset = wmt24pp_dataset_dict["train"]
        wmt24pp_eval_dataset: Dataset = wmt24pp_dataset_dict["test"]
        print("Loaded wmt24pp dataset.")
#
        print("Loading synthia_german_experimental dataset...")
        # source: https://huggingface.co/datasets/jphme/synthia_german_experimental
        # original: https://huggingface.co/datasets/migtissera/Synthia-v1.3
        # License: Apache-2.0
        # interesting fields: instruction, response
        # entries: ~100000
        # final: 14453
        # notes: filtered on scores, take only if all scores are "3" (best).
        synthia_de_ds = load_dataset("jphme/synthia_german_experimental", split="train").filter(lambda _: _["score_deutsch"] == 3 and _["score_antwort"] == 3)
        synthia_de_ds = synthia_de_ds.select_columns(["instruction", "response"])
        synthia_de_ds = synthia_de_ds.train_test_split(test_size=0.02, seed=12)
        synthia_de_train_dataset: Dataset = synthia_de_ds["train"]
        synthia_de_eval_dataset: Dataset = synthia_de_ds["test"]
        print("Loaded synthia_german_experimental dataset.")
#
        print("Loading ger-backtrans-paraphrase dataset...")
        # source: https://huggingface.co/datasets/deutsche-telekom/ger-backtrans-paraphrase
        # License: CC-BY-SA-4.0
        # entries: 21292789
        # filtered: 862574 (tokens >= 25, cos_sim >=0.9)
        # filtered: ~2.1M (tokens >= 17, cos_sim >=0.8) (once a try - results were really bad)
        # notes: also thanks to Daniel Heinze for more filter examples
        # source: https://huggingface.co/datasets/danielheinz/telekom-backtrans-paraphrase-filtered
        # original dataset without hard negatives unused
        #telekom_gbp_dataset = load_dataset("deutsche-telekom/ger-backtrans-paraphrase", split="train")
        #telekom_gbp_dataset = telekom_gbp_dataset.filter(lambda _: _["cos_sim"] >= 0.9 and _["cos_sim"] < 0.999 and _["jaccard_similarity"] >= 0.3  and _["en_de_token_count"] >= 25 and _["de_token_count"] >= 25)
        #telekom_gbp_dataset = telekom_gbp_dataset.select_columns(['en', 'de', 'en_de'])
        # make a copy - but only with 'en_de' and 'de'
        #telekom_gbp_ende_dataset = telekom_gbp_dataset.select_columns(['en_de', 'de'])
        # build the 'original' set
        #telekom_gbp_dataset_dict = telekom_gbp_dataset.train_test_split(test_size=0.05, seed=12)
        #telekom_gbp_train_dataset: Dataset = telekom_gbp_dataset_dict["train"]
        #telekom_gbp_eval_dataset: Dataset = telekom_gbp_dataset_dict["test"]
        # now build a second set of 'bad' to 'good'
        #telekom_gbp_ende_dataset_dict = telekom_gbp_ende_dataset.train_test_split(test_size=0.05, seed=12)
        #telekom_gbp_ende_train_dataset: Dataset = telekom_gbp_ende_dataset_dict["train"]
        #telekom_gbp_ende_eval_dataset: Dataset = telekom_gbp_ende_dataset_dict["test"]
        #
        # FILTERED, SPLIT AND WITH HARD NEGATIVES
        gbp_3hn_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-350c-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        gbp_3hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-200c-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        gbp_3hn_ds = concatenate_datasets([gbp_3hn_ds, gbp_3hn_add_ds])
        gbp_3hn_ds = gbp_3hn_ds.train_test_split(test_size=0.02, seed=12)
        gbp_3hn_train_ds: Dataset = gbp_3hn_ds['train']
        gbp_3hn_eval_ds: Dataset = gbp_3hn_ds['test']
        #
        gbp_0hn_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-350c-sts_3hn/0_hard_negatives/*.parquet'}, split="train")
        gbp_0hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-200c-sts_3hn/0_hard_negatives/*.parquet'}, split="train")
        gbp_0hn_ds = concatenate_datasets([gbp_0hn_ds, gbp_0hn_add_ds])
        gbp_0hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-150c-sts_3hn/0_hard_negatives/*.parquet'}, split="train")
        gbp_0hn_ds = concatenate_datasets([gbp_0hn_ds, gbp_0hn_add_ds])
        gbp_0hn_ds = gbp_0hn_ds.train_test_split(test_size=0.02, seed=12)
        gbp_0hn_train_ds: Dataset = gbp_0hn_ds['train']
        gbp_0hn_eval_ds: Dataset = gbp_0hn_ds['test']
        #
        gbp_ende_3hn_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-en_de-350c-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        gbp_ende_3hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-en_de-200c-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        gbp_ende_3hn_ds = concatenate_datasets([gbp_ende_3hn_ds, gbp_ende_3hn_add_ds])
        gbp_ende_3hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-en_de-150c-sts_3hn/3_hard_negatives/*.parquet'}, split="train")
        gbp_ende_3hn_ds = concatenate_datasets([gbp_ende_3hn_ds, gbp_ende_3hn_add_ds])
        gbp_ende_3hn_ds = gbp_ende_3hn_ds.train_test_split(test_size=0.02, seed=12)
        gbp_ende_3hn_train_ds: Dataset = gbp_ende_3hn_ds['train']
        gbp_ende_3hn_eval_ds: Dataset = gbp_ende_3hn_ds['test']
        #
        gbp_ende_0hn_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-en_de-350c-sts_3hn/0_hard_negatives/*.parquet'}, split="train")
        gbp_ende_0hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-en_de-200c-sts_3hn/0_hard_negatives/*.parquet'}, split="train")
        gbp_ende_0hn_ds = concatenate_datasets([gbp_ende_0hn_ds, gbp_ende_0hn_add_ds])
        gbp_ende_0hn_add_ds = load_dataset('parquet', data_files={'ger-backtrans-paraphrase-en_de-150c-sts_3hn/0_hard_negatives/*.parquet'}, split="train")
        gbp_ende_0hn_ds = concatenate_datasets([gbp_ende_0hn_ds, gbp_ende_0hn_add_ds])
        gbp_ende_0hn_ds = gbp_ende_0hn_ds.train_test_split(test_size=0.02, seed=12)
        gbp_ende_0hn_train_ds: Dataset = gbp_ende_0hn_ds['train']
        gbp_ende_0hn_eval_ds: Dataset = gbp_ende_0hn_ds['test']
        print("Loaded ger-backtrans-paraphrase dataset.")
#
        print("Loading STSb Multi MT (de) dataset...")
        # source: https://huggingface.co/datasets/PhilipMay/stsb_multi_mt
        # License: CC-BY-SA-4.0 - https://github.com/PhilipMay/stsb-multi-mt/blob/main/LICENSE
        # Original: https://ixa2.si.ehu.eus/stswiki/index.php/STSbenchmark
        # entries: 5749
        #stbs_de_dataset = load_dataset("PhilipMay/stsb_multi_mt", "de").filter(lambda _: _["similarity_score"] >= 1 and _["similarity_score"] < 5)
        #stbs_de_dataset = stbs_de_dataset.select_columns(['sentence1', 'sentence2'])
        #stbs_de_train_dataset: Dataset = stbs_de_dataset["train"]
        #stbs_de_eval_dataset: Dataset = stbs_de_dataset["dev"]
        #
        stbs_de_3hn_train_dataset = load_dataset('parquet', data_files={'stsb_multi_mt-de-hn/3_hard_negatives/train*.parquet'}, split="train")
        stbs_de_3hn_eval_dataset = load_dataset('parquet', data_files={'stsb_multi_mt-de-hn/3_hard_negatives/test*.parquet'}, split="train")
        print("Loaded STSb Multi MT (de) dataset.")
#
        print("Loading STSb Multi MT (en) dataset...")
        # source: https://huggingface.co/datasets/PhilipMay/stsb_multi_mt
        # License: CC-BY-SA-4.0 - https://github.com/PhilipMay/stsb-multi-mt/blob/main/LICENSE
        # Original: https://ixa2.si.ehu.eus/stswiki/index.php/STSbenchmark
        # entries: 5749
        #stbs_en_dataset = load_dataset("PhilipMay/stsb_multi_mt", "en").filter(lambda _: _["similarity_score"] >= 1 and _["similarity_score"] < 5)
        #stbs_en_dataset = stbs_en_dataset.select_columns(['sentence1', 'sentence2'])
        #stbs_en_train_dataset: Dataset = stbs_en_dataset["train"]
        #stbs_en_eval_dataset: Dataset = stbs_en_dataset["dev"]
        #
        stbs_en_3hn_train_dataset = load_dataset('parquet', data_files={'stsb_multi_mt-en-hn/3_hard_negatives/train*.parquet'}, split="train")
        stbs_en_3hn_eval_dataset = load_dataset('parquet', data_files={'stsb_multi_mt-en-hn/3_hard_negatives/test*.parquet'}, split="train")
        print("Loaded STSb Multi MT (en) dataset.")
#
        print("Loading paws-x (de) dataset...")
        # source: https://huggingface.co/datasets/google-research-datasets/paws-x
        # License: Other - https://github.com/google-research-datasets/paws/blob/master/LICENSE
        # License: The dataset may be freely used for any purpose, although acknowledgement of Google LLC ("Google") as the data source would be appreciated.
        # entries: 49401
        # Info: filtered only for "true" answers (["label"] == 1)
        pawsx_de_dataset = load_dataset("google-research-datasets/paws-x", "de").filter(lambda _: _["label"] == 1)
        pawsx_de_dataset = pawsx_de_dataset.select_columns(['sentence1', 'sentence2'])
        pawsx_de_train_dataset: Dataset = pawsx_de_dataset["train"]
        pawsx_de_eval_dataset: Dataset = pawsx_de_dataset["validation"]

        print("Loaded paws-x (de) dataset.")
#
        print("Loading paws-x (en) dataset...")
        # source: https://huggingface.co/datasets/google-research-datasets/paws-x
        # License: Other - https://github.com/google-research-datasets/paws/blob/master/LICENSE
        # entries: 49401
        pawsx_en_dataset = load_dataset("google-research-datasets/paws-x", "en").filter(lambda _: _["label"] == 1)
        pawsx_en_dataset = pawsx_en_dataset.select_columns(['sentence1', 'sentence2'])
        pawsx_en_train_dataset: Dataset = pawsx_en_dataset["train"]
        pawsx_en_eval_dataset: Dataset = pawsx_en_dataset["validation"]
        print("Loaded paws-x (en) dataset.")
#
        print("Loading all NLI-26lang-2mil7 (local) datasets...")
        # source: https://huggingface.co/datasets/MoritzLaurer/multilingual-NLI-26lang-2mil7
        # License: MIT
        # License-source: https://github.com/easonnie/combine-FEVER-NSMN
        # entries: 25000
        # info: 'label' – entailment (0), neutral (1), contradiction (2).
        # for simple translations
        main_name = 'multilingual-NLI-26lang-2mil7'
        language = 'de'
        entail = 'de_entailment'
        transl = 'en_de'
        subset = 'anli'
        # anli entailments 3hn - de_anli_entail_3hn_train_ds
        de_anli_entail_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_anli_entail_3hn_train_ds: Dataset = de_anli_entail_3hn_ds['train']
        de_anli_entail_3hn_eval_ds: Dataset = de_anli_entail_3hn_ds['test']
        # anli entailments 0hn - de_anli_entail_0hn_train_ds
        de_anli_entail_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_anli_entail_0hn_train_ds: Dataset = de_anli_entail_0hn_ds['train']
        de_anli_entail_0hn_eval_ds: Dataset = de_anli_entail_0hn_ds['test']
        # anli translation 3hn - de_anli_transl_3hn_train_ds
        de_anli_transl_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_anli_transl_3hn_train_ds: Dataset = de_anli_transl_3hn_ds['train']
        de_anli_transl_3hn_eval_ds: Dataset = de_anli_transl_3hn_ds['test']
        # anli translation 0hn - de_anli_transl_0hn_train_ds
        de_anli_transl_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_anli_transl_0hn_train_ds: Dataset = de_anli_transl_0hn_ds['train']
        de_anli_transl_0hn_eval_ds: Dataset = de_anli_transl_0hn_ds['test']
#
        subset = 'fever'
        # fever entailments 3hn - de_fever_entail_3hn_train_ds
        de_fever_entail_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_fever_entail_3hn_train_ds: Dataset = de_fever_entail_3hn_ds['train']
        de_fever_entail_3hn_eval_ds: Dataset = de_fever_entail_3hn_ds['test']
        # fever entailments 0hn - de_fever_entail_0hn_train_ds
        de_fever_entail_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_fever_entail_0hn_train_ds: Dataset = de_fever_entail_0hn_ds['train']
        de_fever_entail_0hn_eval_ds: Dataset = de_fever_entail_0hn_ds['test']
        # fever translation 3hn - de_fever_transl_3hn_train_ds
        de_fever_transl_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_fever_transl_3hn_train_ds: Dataset = de_fever_transl_3hn_ds['train']
        de_fever_transl_3hn_eval_ds: Dataset = de_fever_transl_3hn_ds['test']
        # fever translation 0hn - de_fever_transl_0hn_train_ds
        de_fever_transl_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_fever_transl_0hn_train_ds: Dataset = de_fever_transl_0hn_ds['train']
        de_fever_transl_0hn_eval_ds: Dataset = de_fever_transl_0hn_ds['test']
#
        subset = 'ling'
        # ling entailments 3hn - de_ling_entail_3hn_train_ds
        de_ling_entail_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_ling_entail_3hn_train_ds: Dataset = de_ling_entail_3hn_ds['train']
        de_ling_entail_3hn_eval_ds: Dataset = de_ling_entail_3hn_ds['test']
        # ling entailments 0hn - de_ling_entail_0hn_train_ds
        de_ling_entail_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_ling_entail_0hn_train_ds: Dataset = de_ling_entail_0hn_ds['train']
        de_ling_entail_0hn_eval_ds: Dataset = de_ling_entail_0hn_ds['test']
        # ling translation 3hn - de_ling_transl_3hn_train_ds
        de_ling_transl_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_ling_transl_3hn_train_ds: Dataset = de_ling_transl_3hn_ds['train']
        de_ling_transl_3hn_eval_ds: Dataset = de_ling_transl_3hn_ds['test']
        # ling translation 0hn - de_ling_transl_0hn_train_ds
        # this set is empty :D
        #de_ling_transl_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        #de_ling_transl_0hn_train_ds: Dataset = de_ling_transl_0hn_ds['train']
        #de_ling_transl_0hn_eval_ds: Dataset = de_ling_transl_0hn_ds['test']
#
        subset = 'mnli'
        # mnli entailments 3hn - de_mnli_entail_3hn_train_ds
        de_mnli_entail_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_mnli_entail_3hn_train_ds: Dataset = de_mnli_entail_3hn_ds['train']
        de_mnli_entail_3hn_eval_ds: Dataset = de_mnli_entail_3hn_ds['test']
        # mnli entailments 0hn - de_mnli_entail_0hn_train_ds
        de_mnli_entail_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_mnli_entail_0hn_train_ds: Dataset = de_mnli_entail_0hn_ds['train']
        de_mnli_entail_0hn_eval_ds: Dataset = de_mnli_entail_0hn_ds['test']
        # mnli translation 3hn - de_mnli_transl_3hn_train_ds
        de_mnli_transl_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_mnli_transl_3hn_train_ds: Dataset = de_mnli_transl_3hn_ds['train']
        de_mnli_transl_3hn_eval_ds: Dataset = de_mnli_transl_3hn_ds['test']
        # mnli translation 0hn - de_mnli_transl_0hn_train_ds
        de_mnli_transl_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_mnli_transl_0hn_train_ds: Dataset = de_mnli_transl_0hn_ds['train']
        de_mnli_transl_0hn_eval_ds: Dataset = de_mnli_transl_0hn_ds['test']
#
        subset = 'wanli'
        # wanli entailments 3hn - de_wanli_entail_3hn_train_ds
        de_wanli_entail_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_wanli_entail_3hn_train_ds: Dataset = de_wanli_entail_3hn_ds['train']
        de_wanli_entail_3hn_eval_ds: Dataset = de_wanli_entail_3hn_ds['test']
        # wanli entailments 0hn - de_wanli_entail_0hn_train_ds
        de_wanli_entail_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{entail}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_wanli_entail_0hn_train_ds: Dataset = de_wanli_entail_0hn_ds['train']
        de_wanli_entail_0hn_eval_ds: Dataset = de_wanli_entail_0hn_ds['test']
        # wanli translation 3hn - de_wanli_transl_3hn_train_ds
        de_wanli_transl_3hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/3_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_wanli_transl_3hn_train_ds: Dataset = de_wanli_transl_3hn_ds['train']
        de_wanli_transl_3hn_eval_ds: Dataset = de_wanli_transl_3hn_ds['test']
        # wanli translation 0hn - de_wanli_transl_0hn_train_ds
        de_wanli_transl_0hn_ds = load_dataset('parquet', data_files={f'{main_name}-{language}_{subset}-{transl}_hn/0_hard_negatives/*.parquet'}, split="train").train_test_split(test_size=0.02, seed=12)
        de_wanli_transl_0hn_train_ds: Dataset = de_wanli_transl_0hn_ds['train']
        de_wanli_transl_0hn_eval_ds: Dataset = de_wanli_transl_0hn_ds['test']
        #
        print("Loaded all NLI-26lang-2mil7 (local hn) datasets...")
#
#       regular dataset unused
        #print("Loading NLI-26lang-2mil7 (anli) dataset...")
        # source: https://huggingface.co/datasets/MoritzLaurer/multilingual-NLI-26lang-2mil7
        # License: MIT
        # License-source: https://github.com/easonnie/combine-FEVER-NSMN
        # entries: 25000
        # info: 'label' – entailment (0), neutral (1), contradiction (2).
        # for simple translations
        #NLI_de_anli_dataset = load_dataset("MoritzLaurer/multilingual-NLI-26lang-2mil7", split="de_anli")
        #NLI_de_anli_ende_dataset = NLI_de_anli_dataset.select_columns(['hypothesis_original', 'hypothesis']).rename_columns({'hypothesis_original': 'sentence1', 'hypothesis': 'sentence2'})
        #NLI_de_anli_ende_dataset2 = NLI_de_anli_dataset.select_columns(['premise_original', 'premise']).rename_columns({'premise_original': 'sentence1', 'premise': 'sentence2'})
        #NLI_de_anli_ende_dataset = concatenate_datasets([NLI_de_anli_ende_dataset, NLI_de_anli_ende_dataset2])
        #del NLI_de_anli_ende_dataset2
        #NLI_de_anli_ende_dataset = NLI_de_anli_ende_dataset.train_test_split(test_size=0.05, seed=12)
        #NLI_de_anli_ende_train_dataset: Dataset = NLI_de_anli_ende_dataset["train"]
        #NLI_de_anli_ende_eval_dataset: Dataset = NLI_de_anli_ende_dataset["test"]
        #
        # for simple entailments from "long" to "conclusion" (like classification)
        #NLI_de_anli_de_entailment_dataset = NLI_de_anli_dataset.select_columns = NLI_de_anli_dataset.filter(lambda _: _["label"] == 0).select_columns(['premise', 'hypothesis']).rename_columns({'premise': 'sentence1', 'hypothesis': 'sentence2'})
        #del NLI_de_anli_dataset
        #NLI_de_anli_de_entailment_dataset = NLI_de_anli_de_entailment_dataset.train_test_split(test_size=0.05, seed=12)
        #NLI_de_anli_entailment_train_dataset: Dataset = NLI_de_anli_de_entailment_dataset["train"]
        #NLI_de_anli_entailment_eval_dataset: Dataset = NLI_de_anli_de_entailment_dataset["test"]
        #print("Loaded NLI-26lang-2mil7 (anli) dataset.")
#
        #print("Loading NLI-26lang-2mil7 (fever) dataset...")
        # source: https://huggingface.co/datasets/MoritzLaurer/multilingual-NLI-26lang-2mil7
        # License: MIT
        # License-source: https://github.com/easonnie/combine-FEVER-NSMN
        # entries: 25000
        #NLI_de_fever_dataset = load_dataset("MoritzLaurer/multilingual-NLI-26lang-2mil7", split="de_fever")
        #NLI_de_fever_dataset2 = NLI_de_fever_dataset
        #NLI_de_fever_dataset3 = NLI_de_fever_dataset.filter(lambda _: _["label"] == 0).select_columns(['premise', 'hypothesis'])
        #NLI_de_fever_dataset = NLI_de_fever_dataset.remove_columns(['label', 'hypothesis_original', 'hypothesis'])
        #NLI_de_fever_dataset2 = NLI_de_fever_dataset2.remove_columns(['label', 'premise_original', 'premise'])
        #NLI_de_fever_dataset2 = NLI_de_fever_dataset2.rename_column('hypothesis_original', 'sentence1')
        #NLI_de_fever_dataset2 = NLI_de_fever_dataset2.rename_column('hypothesis', 'sentence2')
        #NLI_de_fever_dataset3 = NLI_de_fever_dataset3.rename_column('hypothesis', 'sentence2')
        #NLI_de_fever_dataset_dict = NLI_de_fever_dataset.train_test_split(test_size=0.05, seed=12)
        #NLI_de_fever_dataset2_dict = NLI_de_fever_dataset2.train_test_split(test_size=0.05, seed=12)
        #NLI_de_fever_dataset3_dict = NLI_de_fever_dataset3.train_test_split(test_size=0.05, seed=12)
        #NLI_de_fever_train_dataset: Dataset = NLI_de_fever_dataset_dict["train"]
        #NLI_de_fever_eval_dataset: Dataset = NLI_de_fever_dataset_dict["test"]
        #NLI_de_fever_train2_dataset: Dataset = NLI_de_fever_dataset2_dict["train"]
        #NLI_de_fever_eval2_dataset: Dataset = NLI_de_fever_dataset2_dict["test"]
        #NLI_de_fever_train3_dataset: Dataset = NLI_de_fever_dataset3_dict["train"]
        #NLI_de_fever_eval3_dataset: Dataset = NLI_de_fever_dataset3_dict["test"]
        #print("Loaded NLI-26lang-2mil7 (fever) dataset.")
#
        print("Loading Jina AI dataset...")
        # source: https://huggingface.co/datasets/jinaai/parallel-sentences
        # License: Apache-2.0
        # entries: 1000
        # info: sadly JinaAI delivers only 1000 pairs (we know we could do better by …)
        # Info: Multilingual in different columns
        jina_ai_ps_dataset = load_dataset("jinaai/parallel-sentences", split="train")
        jina_ai_ps_dataset_3en = jina_ai_ps_dataset.select_columns(['anchor', 'entailment', 'negative'])
        jina_ai_ps_dataset_en_de = jina_ai_ps_dataset.select_columns(['anchor', 'anchor_de'])
        jina_ai_ps_dataset_de_de = jina_ai_ps_dataset.select_columns(['anchor_de', 'entailment_de'])
        # splits
        jina_ai_ps_dataset_3en_dict = jina_ai_ps_dataset_3en.train_test_split(test_size=0.05, seed=12)
        jina_ai_ps_dataset_en_de_dict = jina_ai_ps_dataset_en_de.train_test_split(test_size=0.05, seed=12)
        jina_ai_ps_dataset_de_de_dict = jina_ai_ps_dataset_de_de.train_test_split(test_size=0.05, seed=12)
        jina_ai_ps_train_3en: Dataset = jina_ai_ps_dataset_3en_dict["train"]
        jina_ai_ps_eval_3en: Dataset = jina_ai_ps_dataset_3en_dict["test"]
        jina_ai_ps_train_en_de: Dataset = jina_ai_ps_dataset_en_de_dict["train"]
        jina_ai_ps_eval_en_de: Dataset = jina_ai_ps_dataset_en_de_dict["test"]
        jina_ai_ps_train_de_de: Dataset = jina_ai_ps_dataset_de_de_dict["train"]
        jina_ai_ps_eval_de_de: Dataset = jina_ai_ps_dataset_de_de_dict["test"]
        print("Loaded Jina AI dataset.")
#
        print("Loading Polyglot-or-Not (de) dataset...")
        # source: https://huggingface.co/datasets/Polyglot-or-Not/Fact-Completion/
        # License: Apache-2.0
        # entries: 16287
        polyglot_de_dataset = load_dataset("Polyglot-or-Not/Fact-Completion", split="German").select_columns(['stem', 'true', 'false'])
        polyglot_de_dict = polyglot_de_dataset.train_test_split(test_size=0.05, seed=12)
        polyglot_de_train_dataset: Dataset = polyglot_de_dict["train"]
        polyglot_de_eval_dataset: Dataset = polyglot_de_dict["test"]
        print("Loaded Polyglot-or-Not (de) dataset.")
#
        print("Loading Polyglot-or-Not (en) dataset...")
        # source: https://huggingface.co/datasets/Polyglot-or-Not/Fact-Completion/
        # License: Apache-2.0
        # entries: 26254
        polyglot_en_dataset = load_dataset("Polyglot-or-Not/Fact-Completion", split="English").select_columns(['stem', 'true', 'false'])
        polyglot_en_dict = polyglot_en_dataset.train_test_split(test_size=0.05, seed=12)
        polyglot_en_train_dataset: Dataset = polyglot_en_dict["train"]
        polyglot_en_eval_dataset: Dataset = polyglot_en_dict["test"]
        print("Loaded Polyglot-or-Not (de) dataset.")
#
        print("Loading Tilde_MODEL_EESC (en_de) dataset...")
        # Tilde MODEL - EESC is a multilingual corpus compiled from document texts of European Economic and Social Committee document portal. Source: http://dm.eesc.europa.eu/
        # License: CC-BY - Creative Commons with Attribution
        # Roberts Rozis, Raivis Skadins, 2017, Tilde MODEL - Multilingual Open Data for EU Languages. Proceedings of the 21th Nordic Conference of Computational Linguistics NODALIDA 2017.
        # https://tilde-model.s3-eu-west-1.amazonaws.com/nodalida2017_Tilde_MODEL.pdf
        # https://tilde-model.s3-eu-west-1.amazonaws.com/Tilde_MODEL_Corpus.html
        #
        # entries: 1860675
        # filtered: 1683698
        # Original (local) version without hard negatives ignored
        #tilde_EESC_dataset = load_dataset("parquet", data_files={'Tilde_MODEL_EESC/EESC.de-en-distilled-scored.parquet.br'}, split='train').filter(lambda _: _['score_sts'] > 0.5 and _['score_sts'] < 1).select_columns(['en', 'de'])
        #tilde_EESC_dataset = tilde_EESC_dataset.train_test_split(test_size=10000, seed=12)
        #tilde_EESC_train_dataset: Dataset = tilde_EESC_dataset["train"]
        #tilde_EESC_eval_dataset: Dataset = tilde_EESC_dataset["test"]
        #del tilde_EESC_dataset
        #
        # loading version with 3 hard negative ignoring folder with 0 negatives
        tilde_EESC_dataset = load_dataset("parquet", data_files={'Tilde_EESC-en-de_hn/3_hard_negatives/train-*.parquet'}, split='train')
        tilde_EESC_dataset = tilde_EESC_dataset.train_test_split(test_size=10000, seed=12)
        tilde_EESC_train_dataset: Dataset = tilde_EESC_dataset["train"]
        tilde_EESC_eval_dataset: Dataset = tilde_EESC_dataset["test"]
        del tilde_EESC_dataset
        #
        print("Loaded Tilde_MODEL_EESC (en_de) dataset.")
#
        print("Loading Tilde_MODEL_RAPID (en_de) dataset...")
        # Tilde MODEL - RAPID multilingual parallel corpus is compiled from all press releases of Press Release Database of European Commission released between 1975 and end of 2016 as available from http://europa.eu/rapid/.
        # License: CC-BY - Creative Commons with Attribution
        # Roberts Rozis, Raivis Skadins, 2017, Tilde MODEL - Multilingual Open Data for EU Languages. Proceedings of the 21th Nordic Conference of Computational Linguistics NODALIDA 2017.
        # https://tilde-model.s3-eu-west-1.amazonaws.com/nodalida2017_Tilde_MODEL.pdf
        # https://tilde-model.s3-eu-west-1.amazonaws.com/Tilde_MODEL_Corpus.html
        #
        # entries: 779236
        # filtered: 727743
        # original scored set needs to be uploaded
        # Original (local) version without hard negatives ignored
        #tilde_RAPID_dataset = load_dataset("parquet", data_files={'Tilde_MODEL_RAPID/RAPID_2019.UNIQUE.de-en-distilled-scored.parquet'}, split='train').filter(lambda _: _['score_sts'] > 0.5 and _['score_sts'] < 1).select_columns(['en', 'de'])
        #tilde_RAPID_dataset = tilde_RAPID_dataset.train_test_split(test_size=10000, seed=12)
        #tilde_RAPID_train_dataset: Dataset = tilde_RAPID_dataset["train"]
        #tilde_RAPID_eval_dataset: Dataset = tilde_RAPID_dataset["test"]
        #del tilde_RAPID_dataset
        #
        # loading version with 3 hard negative ignoring folder with 0 negatives
        tilde_RAPID_dataset = load_dataset("parquet", data_files={'Tilde_RAPID_2019-en-de-hn/3_hard_negatives/train-*.parquet'}, split='train')
        tilde_RAPID_dataset = tilde_RAPID_dataset.train_test_split(test_size=10000, seed=12)
        tilde_RAPID_train_dataset: Dataset = tilde_RAPID_dataset["train"]
        tilde_RAPID_eval_dataset: Dataset = tilde_RAPID_dataset["test"]
        del tilde_RAPID_dataset
        print("Loaded Tilde_MODEL_RAPID (en_de) dataset.")
#
        print("Loading miracl (de) as classification dataset...")
        miracl_de_dataset = load_dataset('parquet', data_files={'miracl-corpus-de-hn-*/3_hard_negatives/train-*.parquet'}, split='train')
        miracl_de_dataset = miracl_de_dataset.train_test_split(test_size=10000, seed=12)
        miracl_de_train_dataset: Dataset = miracl_de_dataset["train"]
        miracl_de_eval_dataset: Dataset = miracl_de_dataset["test"]
        #
        miracl_de_0hn_dataset = load_dataset('parquet', data_files={'miracl-corpus-de-hn_hn/0_hard_negatives/train-*.parquet'}, split='train')
        miracl_de_0hn_dataset = miracl_de_0hn_dataset.train_test_split(test_size=0.02, seed=12)
        miracl_de_0hn_train_dataset: Dataset = miracl_de_0hn_dataset['train']
        miracl_de_0hn_eval_dataset: Dataset = miracl_de_0hn_dataset['test']
        print("Loaded miracl (de) as classification dataset.")
#
        train_dataset = DatasetDict({
            'mmarco_3hn': mmarco_de_3hn_train_dataset,
            'mmarco_2hn': mmarco_de_2hn_train_dataset,
            'mmarco_1hn': mmarco_de_1hn_train_dataset,
            'mmarco_0hn': mmarco_de_0hn_train_dataset,
            'wp-22-12-de': wp_2212_de_train_dataset,
            #'wp-22-12-de_3hn': wp_2212_de_train_dataset,
            #'wp-22-12-de_0hn': wp_2212_de_0_train_dataset,
            'swim_ir_de': swim_ir_de_train_dataset,
            'swim_ir_de_3hn': swim_ir_de_3hn_train_dataset,
            'swim_ir_de_title_3hn': swim_ir_de_title_3hn_train_dataset,
            'swim_ir_de_title': swim_ir_de_title_train_dataset,
            'avemio_triples': avemio_triples_train_dataset,
            'avemio_pairs_3hn': avemio_pairs_3hn_train_ds,
            'avemio_pairs_0hn': avemio_pairs_0hn_train_ds,
            'nq_german_en_de_a_3hn': nq_german_en_de_a_3hn_train_ds,
            'nq_german_en_de_3hn': nq_german_en_de_3hn_train_ds,
            'nq_german_3hn': nq_german_3hn_train_ds,
            'nq_german_1hn': nq_german_1hn_train_ds,
            #'german_oasst1': german_oasst1_train_dataset,
            'german_oasst1_hn': german_oasst1_hn_train_dataset,
            'germanrag_short': germanrag_short_train_dataset,
            'slimorca_dedup_3hn': slimorca_dedup_3hn_train_ds,
            'slimorca_dedup_2hn': slimorca_dedup_2hn_train_ds,
            'slimorca_dedup_1hn': slimorca_dedup_1hn_train_ds,
            'slimorca_dedup_0hn': slimorca_dedup_0hn_train_ds,
            #'german_gpt4': german_gpt4_train_dataset,
            'german_gpt4_3hn': german_gpt4_3hn_train_dataset,
            'german_orca_dpo': german_orca_dpo_train_dataset,
            'alpaca_gpt4_3hn': alpaca_gpt4_de_3hn_train_dataset,
            'alpaca_gpt4_0hn': alpaca_gpt4_de_0hn_train_dataset,
            'dolly_context_de_3hn': dolly_context_de_3hn_train_ds,
            #'dolly_context_de_0hn': dolly_context_de_0hn_train_ds,
            'dolly_context_ende_3hn': dolly_context_ende_3hn_train_ds,
            'dolly_instructions_de_3hn': dolly_instructions_de_3hn_train_ds,
            'dolly_instructions_de_0hn': dolly_instructions_de_0hn_train_ds,
            'dolly_instructions_ende_3hn': dolly_instructions_ende_3hn_train_ds,
            #'dolly_instructions_ende_0hn': dolly_instructions_ende_0hn_train_ds,
            'dolly_responses_de_3hn': dolly_responses_de_3hn_train_ds,
            'dolly_responses_de_0hn': dolly_responses_de_0hn_train_ds,
            'dolly_responses_ende_3hn': dolly_responses_ende_3hn_train_ds,
            #'dolly_responses_ende_0hn': dolly_responses_ende_0hn_train_ds,
            'saf_legal_de': saf_legal_de_train_ds,
            'gls_3hn': gls_3hn_train_dataset,
            'gls_2hn': gls_2hn_train_dataset,
            'gls_1hn': gls_1hn_train_dataset,
            'gls_0hn': gls_0hn_train_dataset,
            'europarl_3hn': europarl_3hn_train_dataset,
            'europarl_0hn': europarl_0hn_train_dataset,
            #'tatoeba': tatoeba_train_dataset,
            'tatoeba_3hn': tatoeba_3hn_train_dataset,
            'tatoeba_0hn': tatoeba_0hn_train_dataset,
            'wikimatrix_3hn': wikimatrix_3hn_train_ds,
            #'wikimatrix_0hn': wikimatrix_0hn_train_ds,
            'wikipedia_abstract_3hn': wikipedia_abstract_3hn_train_dataset,
            'wikipedia_abstract_0hn': wikipedia_abstract_0hn_train_dataset,
            'wiktionary_gdg_de_3hn': wiktionary_gdg_de_3hn_train_ds,
            'wiktionary_gdg_de_short': wiktionary_gdg_de_short_train_dataset,
            'wmt24pp': wmt24pp_train_dataset,
            'synthia_de': synthia_de_train_dataset,
            'gbp_3hn': gbp_3hn_train_ds,
            #'gbp_0hn': gbp_0hn_train_ds,
            'gbp_ende_3hn': gbp_ende_3hn_train_ds,
            #'gbp_ende_0hn': gbp_ende_0hn_train_ds,
            #'stbs_de': stbs_de_train_dataset,
            'stbs_de_3hn': stbs_de_3hn_train_dataset,
            #'stbs_en': stbs_en_train_dataset,
            'stbs_en_3hn': stbs_en_3hn_train_dataset,
            'pawsx_de': pawsx_de_train_dataset,
            'pawsx_en': pawsx_en_train_dataset,
            'nli_anli_entail_3hn': de_anli_entail_3hn_train_ds,
            'nli_fever_entail_3hn': de_fever_entail_3hn_train_ds,
            'nli_ling_entail_3hn': de_ling_entail_3hn_train_ds,
            'nli_mnli_entail_3hn': de_mnli_entail_3hn_train_ds,
            'nli_wanli_entail_3hn': de_wanli_entail_3hn_train_ds,
            #'nli_anli_entail_0hn': de_anli_entail_0hn_train_ds,
            #'nli_fever_entail_0hn': de_fever_entail_0hn_train_ds,
            #'nli_ling_entail_0hn': de_ling_entail_0hn_train_ds,
            #'nli_mnli_entail_0hn': de_mnli_entail_0hn_train_ds,
            #'nli_wanli_entail_0hn': de_wanli_entail_0hn_train_ds,
            'nli_anli_transl_3hn': de_anli_transl_3hn_train_ds,
            'nli_fever_transl_3hn': de_fever_transl_3hn_train_ds,
            'nli_ling_transl_3hn': de_ling_transl_3hn_train_ds,
            'nli_mnli_transl_3hn': de_mnli_transl_3hn_train_ds,
            'nli_wanli_transl_3hn': de_wanli_transl_3hn_train_ds,
            #'nli_anli_transl_0hn': de_anli_transl_0hn_train_ds,
            #'nli_fever_transl_0hn': de_fever_transl_0hn_train_ds,
            #'nli_ling_transl_0hn': de_ling_transl_0hn_train_ds,
            #'nli_mnli_transl_0hn': de_mnli_transl_0hn_train_ds,
            #'nli_wanli_transl_0hn': de_wanli_transl_0hn_train_ds,
            'jina_ai_3en': jina_ai_ps_train_3en,
            'jina_ai_ende': jina_ai_ps_train_en_de,
            'jina_ai_dede': jina_ai_ps_train_de_de,
            'polyglot_de': polyglot_de_train_dataset,
            'polyglot_en': polyglot_en_train_dataset,
            'tilde_EESC': tilde_EESC_train_dataset,
            #'tilde_RAPID': tilde_RAPID_train_dataset,
            'miracl_de_3hn': miracl_de_train_dataset,
            'miracl_de_0hn': miracl_de_0hn_train_dataset,
        })
        eval_dataset = DatasetDict({
            'mmarco_3hn': mmarco_de_3hn_eval_dataset,
            'mmarco_2hn': mmarco_de_2hn_eval_dataset,
            'mmarco_1hn': mmarco_de_1hn_eval_dataset,
            'mmarco_0hn': mmarco_de_0hn_eval_dataset,
            'wp-22-12-de': wp_2212_de_eval_dataset,
            #'wp-22-12-de_3hn': wp_2212_de_eval_dataset,
            #'wp-22-12-de_0hn': wp_2212_de_0_eval_dataset,
            'swim_ir_de': swim_ir_de_eval_dataset,
            'swim_ir_de_3hn': swim_ir_de_3hn_eval_dataset,
            'swim_ir_de_title_3hn': swim_ir_de_title_3hn_eval_dataset,
            'swim_ir_de_title': swim_ir_de_title_eval_dataset,
            'avemio_triples': avemio_triples_eval_dataset,
            'avemio_pairs_3hn': avemio_pairs_3hn_eval_ds,
            'avemio_pairs_0hn': avemio_pairs_0hn_eval_ds,
            'nq_german_en_de_a_3hn': nq_german_en_de_a_3hn_eval_ds,
            'nq_german_en_de_3hn': nq_german_en_de_3hn_eval_ds,
            'nq_german_3hn': nq_german_3hn_eval_ds,
            'nq_german_1hn': nq_german_1hn_eval_ds,
            #'german_oasst1': german_oasst1_eval_dataset,
            'german_oasst1_hn': german_oasst1_hn_eval_dataset,
            'germanrag_short': germanrag_short_eval_dataset,
            'slimorca_dedup_3hn': slimorca_dedup_3hn_eval_ds,
            'slimorca_dedup_2hn': slimorca_dedup_2hn_eval_ds,
            'slimorca_dedup_1hn': slimorca_dedup_1hn_eval_ds,
            'slimorca_dedup_0hn': slimorca_dedup_0hn_eval_ds,
            #'german_gpt4': german_gpt4_eval_dataset,
            'german_gpt4_3hn': german_gpt4_3hn_eval_dataset,
            'german_orca_dpo': german_orca_dpo_eval_dataset,
            'alpaca_gpt4_3hn': alpaca_gpt4_de_3hn_eval_dataset,
            'alpaca_gpt4_0hn': alpaca_gpt4_de_0hn_eval_dataset,
            'dolly_context_de_3hn': dolly_context_de_3hn_eval_ds,
            #'dolly_context_de_0hn': dolly_context_de_0hn_eval_ds,
            'dolly_context_ende_3hn': dolly_context_ende_3hn_eval_ds,
            'dolly_instructions_de_3hn': dolly_instructions_de_3hn_eval_ds,
            'dolly_instructions_de_0hn': dolly_instructions_de_0hn_eval_ds,
            'dolly_instructions_ende_3hn': dolly_instructions_ende_3hn_eval_ds,
            #'dolly_instructions_ende_0hn': dolly_instructions_ende_0hn_eval_ds,
            'dolly_responses_de_3hn': dolly_responses_de_3hn_eval_ds,
            'dolly_responses_de_0hn': dolly_responses_de_0hn_eval_ds,
            'dolly_responses_ende_3hn': dolly_responses_ende_3hn_eval_ds,
            #'dolly_responses_ende_0hn': dolly_responses_ende_0hn_eval_ds,
            'saf_legal_de': saf_legal_de_eval_ds,
            'gls_3hn': gls_3hn_eval_dataset,
            'gls_2hn': gls_2hn_eval_dataset,
            'gls_1hn': gls_1hn_eval_dataset,
            'gls_0hn': gls_0hn_eval_dataset,
            'europarl_3hn': europarl_3hn_eval_dataset,
            'europarl_0hn': europarl_0hn_eval_dataset,
            #'tatoeba': tatoeba_eval_dataset,
            'tatoeba_3hn': tatoeba_3hn_eval_dataset,
            'tatoeba_0hn': tatoeba_0hn_eval_dataset,
            'wikimatrix_3hn': wikimatrix_3hn_eval_ds,
            #'wikimatrix_0hn': wikimatrix_0hn_eval_ds,
            'wikipedia_abstract_3hn': wikipedia_abstract_3hn_eval_dataset,
            'wikipedia_abstract_0hn': wikipedia_abstract_0hn_eval_dataset,
            'wiktionary_gdg_de_3hn': wiktionary_gdg_de_3hn_eval_ds,
            'wiktionary_gdg_de_short': wiktionary_gdg_de_short_eval_dataset,
            'wmt24pp': wmt24pp_eval_dataset,
            'synthia_de': synthia_de_eval_dataset,
            'gbp_3hn': gbp_3hn_eval_ds,
            #'gbp_0hn': gbp_0hn_eval_ds,
            'gbp_ende_3hn': gbp_ende_3hn_eval_ds,
            #'gbp_ende_0hn': gbp_ende_0hn_eval_ds,
            #'stbs_de': stbs_de_eval_dataset,
            'stbs_de_3hn': stbs_de_3hn_eval_dataset,
            #'stbs_en': stbs_en_eval_dataset,
            'stbs_en_3hn': stbs_en_3hn_eval_dataset,
            'pawsx_de': pawsx_de_eval_dataset,
            'pawsx_en': pawsx_en_eval_dataset,
            'nli_anli_entail_3hn': de_anli_entail_3hn_eval_ds,
            'nli_fever_entail_3hn': de_fever_entail_3hn_eval_ds,
            'nli_ling_entail_3hn': de_ling_entail_3hn_eval_ds,
            'nli_mnli_entail_3hn': de_mnli_entail_3hn_eval_ds,
            'nli_wanli_entail_3hn': de_wanli_entail_3hn_eval_ds,
            #'nli_anli_entail_0hn': de_anli_entail_0hn_eval_ds,
            #'nli_fever_entail_0hn': de_fever_entail_0hn_eval_ds,
            #'nli_ling_entail_0hn': de_ling_entail_0hn_eval_ds,
            #'nli_mnli_entail_0hn': de_mnli_entail_0hn_eval_ds,
            #'nli_wanli_entail_0hn': de_wanli_entail_0hn_eval_ds,
            'nli_anli_transl_3hn': de_anli_transl_3hn_eval_ds,
            'nli_fever_transl_3hn': de_fever_transl_3hn_eval_ds,
            'nli_ling_transl_3hn': de_ling_transl_3hn_eval_ds,
            'nli_mnli_transl_3hn': de_mnli_transl_3hn_eval_ds,
            'nli_wanli_transl_3hn': de_wanli_transl_3hn_eval_ds,
            #'nli_anli_transl_0hn': de_anli_transl_0hn_eval_ds,
            #'nli_fever_transl_0hn': de_fever_transl_0hn_eval_ds,
            #'nli_ling_transl_0hn': de_ling_transl_0hn_eval_ds,
            #'nli_mnli_transl_0hn': de_mnli_transl_0hn_eval_ds,
            #'nli_wanli_transl_0hn': de_wanli_transl_0hn_eval_ds,
            'jina_ai_3en': jina_ai_ps_eval_3en,
            'jina_ai_ende': jina_ai_ps_eval_en_de,
            'jina_ai_dede': jina_ai_ps_eval_de_de,
            'polyglot_de': polyglot_de_eval_dataset,
            'polyglot_en': polyglot_en_eval_dataset,
            'tilde_EESC': tilde_EESC_eval_dataset,
            #'tilde_RAPID': tilde_RAPID_eval_dataset,
            'miracl_de_3hn': miracl_de_eval_dataset,
            'miracl_de_0hn': miracl_de_0hn_eval_dataset,
        })
#
        train_dataset.save_to_disk("base_datasets/train_dataset")
        eval_dataset.save_to_disk("base_datasets/eval_dataset")
#
        end_time = timer()
        print('Time for preprocessing (minutes): '+str(round((end_time - start_time)/60, 3))) # the cheapest full timer one can get.
# The `train_test_split` calls have put a lot of the datasets in memory, while we want it to just be on disk
        # So we're calling quit() here. Running the script again will load the datasets from disk.
        quit()

def main():
    # 1. Load a model to finetune with 2. (Optional) model card data
    static_embedding = StaticEmbedding(AutoTokenizer.from_pretrained(f"{tokenizer_model}"), embedding_dim=2048)
    model = SentenceTransformer(
        modules=[static_embedding],
        model_card_data=SentenceTransformerModelCardData(
            language="de, en",
            license="eupl-1.2",
            model_name=f"A static embedding model tokenized with {tokenizer_model} and mainly built on DE/EN-datasets.",
        ),
    )
#
    # 3. Set up training & evaluation datasets - each dataset is trained with MNRL (with MRL)
    train_dataset, eval_dataset = load_train_eval_datasets()
    print(train_dataset)
#
    # 4. Define a loss function
    # sadly at the moment neither CachedMultipleNegativesRankingLoss or GISTEmbedLoss work with StaticEmbedding.
    loss = MultipleNegativesRankingLoss(model)
    loss = MatryoshkaLoss(model, loss, matryoshka_dims=[32, 64, 128, 256, 512, 1024, 2048])
#
    # 5. (Optional) Specify training arguments
    # check for GPU support (using already loaded tensorflow)
    if len(tf.config.list_physical_devices('GPU')) > 0:
        fp16=True
        bf16=False
    else:
        fp16=False
        bf16=True
    ## manual override
    #fp16=False
    #bf16=False
    run_name = f"{sts_basename}-v{version}"
    args = SentenceTransformerTrainingArguments(
        # Required parameter:
        output_dir=f"models/{run_name}",
        # Optional training parameters:
        num_train_epochs=1, # original 1 - if 2 epochs deliver worse results, it's already overfitting.
        per_device_train_batch_size=1024 * 4, # original 2048 - suggestions are 16384 (but beware of the GPU-RAM(!))
        per_device_eval_batch_size=1024 * 4, # original 2048
        learning_rate=2e-1,
        lr_scheduler_type="cosine", # instead of 'linear'
        warmup_ratio=0.1,
        fp16=fp16,  # Set to False if you get an error that your GPU can't run on FP16
        bf16=bf16,  # Set to True if you have a GPU that supports BF16
        batch_sampler=BatchSamplers.NO_DUPLICATES,  # MultipleNegativesRankingLoss benefits from no duplicate samples in a batch
        multi_dataset_batch_sampler=MultiDatasetBatchSamplers.PROPORTIONAL,
        # Optional tracking/debugging parameters:
        eval_strategy="steps",
        eval_steps=500,
        save_strategy="steps",
        save_steps=1000,
        save_total_limit=2,
        logging_steps=500,
        logging_first_step=True,
        run_name=run_name,  # Will be used in W&B if `wandb` is installed
    )
#
    # 6. Create a trainer & train
    trainer = SentenceTransformerTrainer(
        model=model,
        args=args,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        loss=loss,
    )
    trainer.train()
#
    # 7. Save the trained model
    model.save_pretrained(f"models/{run_name}/final")
#
    # 8. (Optional) Push it to the Hugging Face Hub
    #model.push_to_hub(run_name, private=True)
#
    # 9. Quick testing the model with NanoBEIR
    ## found at: https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#nanobeirevaluator
    evaluator = NanoBEIREvaluator(show_progress_bar=True)
    results = evaluator(model)
    print('\n' + str(results[evaluator.primary_metric]))

# STARTER
if __name__ == "__main__":
    start_time = timer()
    main()
    end_time = timer()
    print('Time for training (minutes): '+str(round((end_time - start_time)/60, 3))) # the cheapest full timer one can get.