mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-31 02:02:21 +06:00
c754c41c61
* added rag WIP * path fix * Formatting / renaming prior to actual work * added rag WIP * path fix * Formatting / renaming prior to actual work * added rag WIP * path fix * Formatting / renaming prior to actual work * added rag WIP * Formatting / renaming prior to actual work * First commit * improve comments * Retrieval evaluation scripts * refactor to include modeling outputs + MPI retriever * Fix rag-token model + refactor * Various fixes + finetuning logic * use_bos fix * Retrieval refactor * Finetuning refactoring and cleanup * Add documentation and cleanup * Remove set_up_rag_env.sh file * Fix retrieval wit HF index * Fix import errors * Fix quality errors * Refactor as per suggestions in https://github.com/huggingface/transformers/pull/6813#issuecomment-687208867 * fix quality * Fix RAG Sequence generation * minor cleanup plus initial tests * fix test * fix tests 2 * Comments fix * post-merge fixes * Improve readme + post-rebase refactor * Extra dependencied for tests * Fix tests * Fix tests 2 * Refactor test requirements * Fix tests 3 * Post-rebase refactor * rename nlp->datasets * RAG integration tests * add tokenizer to slow integration test and allow retriever to run on cpu * add tests; fix position ids warning * change structure * change structure * add from encoder generator * save working solution * make all integration tests pass * add RagTokenizer.save/from_pretrained and RagRetriever.save/from_pretrained * don't save paths * delete unnecessary imports * pass config to AutoTokenizer.from_pretrained for Rag tokenizers * init wiki_dpr only once * hardcode legacy index and passages paths (todo: add the right urls) * finalize config * finalize retriver api and config api * LegacyIndex index download refactor * add dpr to autotokenizer * make from pretrained more flexible * fix ragfortokengeneration * small name changes in tokenizer * add labels to models * change default index name * add retrieval tests * finish token generate * align test with previous version and make all tests pass * add tests * finalize tests * implement thoms suggestions * add first version of test * make first tests work * make retriever platform agnostic * naming * style * add legacy index URL * docstrings + simple retrieval test for distributed * clean model api * add doc_ids to retriever's outputs * fix retrieval tests * finish model outputs * finalize model api * fix generate problem for rag * fix generate for other modles * fix some tests * save intermediate * set generate to default * big refactor generate * delete rag_api * correct pip faiss install * fix auto tokenization test * fix faiss install * fix test * move the distributed logic to examples * model page * docs * finish tests * fix dependencies * fix import in __init__ * Refactor eval_rag and finetune scripts * start docstring * add psutil to test * fix tf test * move require torch to top * fix retrieval test * align naming * finish automodel * fix repo consistency * test ragtokenizer save/load * add rag model output docs * fix ragtokenizer save/load from pretrained * fix tokenizer dir * remove torch in retrieval * fix docs * fixe finetune scripts * finish model docs * finish docs * remove auto model for now * add require torch * remove solved todos * integrate sylvains suggestions * sams comments * correct mistake on purpose * improve README * Add generation test cases * fix rag token * clean token generate * fix test * add note to test * fix attention mask * add t5 test for rag * Fix handling prefix in finetune.py * don't overwrite index_name Co-authored-by: Patrick Lewis <plewis@fb.com> Co-authored-by: Aleksandra Piktus <piktus@devfair0141.h2.fair> Co-authored-by: Aleksandra Piktus <piktus@learnfair5102.h2.fair> Co-authored-by: Aleksandra Piktus <piktus@learnfair5067.h2.fair> Co-authored-by: Your Name <you@example.com> Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com> Co-authored-by: Quentin Lhoest <lhoest.q@gmail.com> |
||
|---|---|---|
| .. | ||
| fixtures | ||
| __init__.py | ||
| conftest.py | ||
| test_activations_tf.py | ||
| test_activations.py | ||
| test_benchmark_tf.py | ||
| test_benchmark.py | ||
| test_cli.py | ||
| test_configuration_auto.py | ||
| test_configuration_common.py | ||
| test_data_collator.py | ||
| test_doc_samples.py | ||
| test_hf_api.py | ||
| test_hf_argparser.py | ||
| test_logging.py | ||
| test_model_card.py | ||
| test_model_output.py | ||
| test_modeling_albert.py | ||
| test_modeling_auto.py | ||
| test_modeling_bart.py | ||
| test_modeling_bert_generation.py | ||
| test_modeling_bert.py | ||
| test_modeling_camembert.py | ||
| test_modeling_common.py | ||
| test_modeling_ctrl.py | ||
| test_modeling_distilbert.py | ||
| test_modeling_dpr.py | ||
| test_modeling_electra.py | ||
| test_modeling_encoder_decoder.py | ||
| test_modeling_flaubert.py | ||
| test_modeling_fsmt.py | ||
| test_modeling_funnel.py | ||
| test_modeling_gpt2.py | ||
| test_modeling_layoutlm.py | ||
| test_modeling_longformer.py | ||
| test_modeling_lxmert.py | ||
| test_modeling_marian.py | ||
| test_modeling_mbart.py | ||
| test_modeling_mobilebert.py | ||
| test_modeling_openai.py | ||
| test_modeling_pegasus.py | ||
| test_modeling_rag.py | ||
| test_modeling_reformer.py | ||
| test_modeling_roberta.py | ||
| test_modeling_t5.py | ||
| test_modeling_tf_albert.py | ||
| test_modeling_tf_auto.py | ||
| test_modeling_tf_bert.py | ||
| test_modeling_tf_camembert.py | ||
| test_modeling_tf_common.py | ||
| test_modeling_tf_ctrl.py | ||
| test_modeling_tf_distilbert.py | ||
| test_modeling_tf_electra.py | ||
| test_modeling_tf_flaubert.py | ||
| test_modeling_tf_funnel.py | ||
| test_modeling_tf_gpt2.py | ||
| test_modeling_tf_longformer.py | ||
| test_modeling_tf_lxmert.py | ||
| test_modeling_tf_mobilebert.py | ||
| test_modeling_tf_openai.py | ||
| test_modeling_tf_roberta.py | ||
| test_modeling_tf_t5.py | ||
| test_modeling_tf_transfo_xl.py | ||
| test_modeling_tf_xlm_roberta.py | ||
| test_modeling_tf_xlm.py | ||
| test_modeling_tf_xlnet.py | ||
| test_modeling_transfo_xl.py | ||
| test_modeling_xlm_roberta.py | ||
| test_modeling_xlm.py | ||
| test_modeling_xlnet.py | ||
| test_onnx.py | ||
| test_optimization_tf.py | ||
| test_optimization.py | ||
| test_pipelines.py | ||
| test_retrieval_rag.py | ||
| test_tokenization_albert.py | ||
| test_tokenization_auto.py | ||
| test_tokenization_bart.py | ||
| test_tokenization_bert_generation.py | ||
| test_tokenization_bert_japanese.py | ||
| test_tokenization_bert.py | ||
| test_tokenization_bertweet.py | ||
| test_tokenization_common.py | ||
| test_tokenization_ctrl.py | ||
| test_tokenization_distilbert.py | ||
| test_tokenization_dpr.py | ||
| test_tokenization_fast.py | ||
| test_tokenization_fsmt.py | ||
| test_tokenization_funnel.py | ||
| test_tokenization_gpt2.py | ||
| test_tokenization_layoutlm.py | ||
| test_tokenization_lxmert.py | ||
| test_tokenization_marian.py | ||
| test_tokenization_mbart.py | ||
| test_tokenization_openai.py | ||
| test_tokenization_pegasus.py | ||
| test_tokenization_phobert.py | ||
| test_tokenization_rag.py | ||
| test_tokenization_reformer.py | ||
| test_tokenization_roberta.py | ||
| test_tokenization_t5.py | ||
| test_tokenization_transfo_xl.py | ||
| test_tokenization_utils.py | ||
| test_tokenization_xlm_roberta.py | ||
| test_tokenization_xlm.py | ||
| test_tokenization_xlnet.py | ||
| test_trainer_distributed.py | ||
| test_trainer.py | ||
| test_utils_check_copies.py | ||