diff --git a/.circleci/config.yml b/.circleci/config.yml index 9cc18139b9b..1ad01c489a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,9 +97,9 @@ jobs: - run: sudo pip install --editable . - run: sudo pip install torch tensorflow - run: sudo pip install black git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort flake8 - - run: black --check --line-length 119 examples templates tests transformers utils - - run: isort --check-only --recursive examples templates tests transformers utils - - run: flake8 examples templates tests transformers utils + - run: black --check --line-length 119 examples templates tests src utils + - run: isort --check-only --recursive examples templates tests src utils + - run: flake8 examples templates tests src utils check_repository_consistency: working_directory: ~/transformers docker: diff --git a/Makefile b/Makefile index e393cdb88a9..b93cd3134a6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: style style: - black --line-length 119 examples templates tests transformers utils - isort --recursive examples templates tests transformers utils + black --line-length 119 examples templates tests src utils + isort --recursive examples templates tests src utils diff --git a/setup.py b/setup.py index 13fe6d90f18..6e9dbba2615 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,8 @@ setup( keywords="NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU", license="Apache", url="https://github.com/huggingface/transformers", - packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), + package_dir = {'': 'src'}, + packages=find_packages("src"), install_requires=[ "numpy", "boto3", diff --git a/transformers/__init__.py b/src/transformers/__init__.py similarity index 100% rename from transformers/__init__.py rename to src/transformers/__init__.py diff --git a/transformers/__main__.py b/src/transformers/__main__.py similarity index 100% rename from transformers/__main__.py rename to src/transformers/__main__.py diff --git a/transformers/commands/__init__.py b/src/transformers/commands/__init__.py similarity index 100% rename from transformers/commands/__init__.py rename to src/transformers/commands/__init__.py diff --git a/transformers/commands/convert.py b/src/transformers/commands/convert.py similarity index 100% rename from transformers/commands/convert.py rename to src/transformers/commands/convert.py diff --git a/transformers/commands/download.py b/src/transformers/commands/download.py similarity index 100% rename from transformers/commands/download.py rename to src/transformers/commands/download.py diff --git a/transformers/commands/run.py b/src/transformers/commands/run.py similarity index 100% rename from transformers/commands/run.py rename to src/transformers/commands/run.py diff --git a/transformers/commands/serving.py b/src/transformers/commands/serving.py similarity index 100% rename from transformers/commands/serving.py rename to src/transformers/commands/serving.py diff --git a/transformers/commands/train.py b/src/transformers/commands/train.py similarity index 100% rename from transformers/commands/train.py rename to src/transformers/commands/train.py diff --git a/transformers/commands/user.py b/src/transformers/commands/user.py similarity index 100% rename from transformers/commands/user.py rename to src/transformers/commands/user.py diff --git a/transformers/configuration_albert.py b/src/transformers/configuration_albert.py similarity index 100% rename from transformers/configuration_albert.py rename to src/transformers/configuration_albert.py diff --git a/transformers/configuration_auto.py b/src/transformers/configuration_auto.py similarity index 100% rename from transformers/configuration_auto.py rename to src/transformers/configuration_auto.py diff --git a/transformers/configuration_bert.py b/src/transformers/configuration_bert.py similarity index 100% rename from transformers/configuration_bert.py rename to src/transformers/configuration_bert.py diff --git a/transformers/configuration_camembert.py b/src/transformers/configuration_camembert.py similarity index 100% rename from transformers/configuration_camembert.py rename to src/transformers/configuration_camembert.py diff --git a/transformers/configuration_ctrl.py b/src/transformers/configuration_ctrl.py similarity index 100% rename from transformers/configuration_ctrl.py rename to src/transformers/configuration_ctrl.py diff --git a/transformers/configuration_distilbert.py b/src/transformers/configuration_distilbert.py similarity index 100% rename from transformers/configuration_distilbert.py rename to src/transformers/configuration_distilbert.py diff --git a/transformers/configuration_gpt2.py b/src/transformers/configuration_gpt2.py similarity index 100% rename from transformers/configuration_gpt2.py rename to src/transformers/configuration_gpt2.py diff --git a/transformers/configuration_mmbt.py b/src/transformers/configuration_mmbt.py similarity index 100% rename from transformers/configuration_mmbt.py rename to src/transformers/configuration_mmbt.py diff --git a/transformers/configuration_openai.py b/src/transformers/configuration_openai.py similarity index 100% rename from transformers/configuration_openai.py rename to src/transformers/configuration_openai.py diff --git a/transformers/configuration_roberta.py b/src/transformers/configuration_roberta.py similarity index 100% rename from transformers/configuration_roberta.py rename to src/transformers/configuration_roberta.py diff --git a/transformers/configuration_t5.py b/src/transformers/configuration_t5.py similarity index 100% rename from transformers/configuration_t5.py rename to src/transformers/configuration_t5.py diff --git a/transformers/configuration_transfo_xl.py b/src/transformers/configuration_transfo_xl.py similarity index 100% rename from transformers/configuration_transfo_xl.py rename to src/transformers/configuration_transfo_xl.py diff --git a/transformers/configuration_utils.py b/src/transformers/configuration_utils.py similarity index 100% rename from transformers/configuration_utils.py rename to src/transformers/configuration_utils.py diff --git a/transformers/configuration_xlm.py b/src/transformers/configuration_xlm.py similarity index 100% rename from transformers/configuration_xlm.py rename to src/transformers/configuration_xlm.py diff --git a/transformers/configuration_xlm_roberta.py b/src/transformers/configuration_xlm_roberta.py similarity index 100% rename from transformers/configuration_xlm_roberta.py rename to src/transformers/configuration_xlm_roberta.py diff --git a/transformers/configuration_xlnet.py b/src/transformers/configuration_xlnet.py similarity index 100% rename from transformers/configuration_xlnet.py rename to src/transformers/configuration_xlnet.py diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_albert_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/convert_bert_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_bert_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_bert_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_bert_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/convert_bert_pytorch_checkpoint_to_original_tf.py b/src/transformers/convert_bert_pytorch_checkpoint_to_original_tf.py similarity index 100% rename from transformers/convert_bert_pytorch_checkpoint_to_original_tf.py rename to src/transformers/convert_bert_pytorch_checkpoint_to_original_tf.py diff --git a/transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/convert_openai_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_openai_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_openai_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_openai_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/src/transformers/convert_pytorch_checkpoint_to_tf2.py similarity index 100% rename from transformers/convert_pytorch_checkpoint_to_tf2.py rename to src/transformers/convert_pytorch_checkpoint_to_tf2.py diff --git a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py rename to src/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py diff --git a/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_t5_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py rename to src/transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py diff --git a/transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py b/src/transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py rename to src/transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py diff --git a/transformers/data/__init__.py b/src/transformers/data/__init__.py similarity index 100% rename from transformers/data/__init__.py rename to src/transformers/data/__init__.py diff --git a/transformers/data/metrics/__init__.py b/src/transformers/data/metrics/__init__.py similarity index 100% rename from transformers/data/metrics/__init__.py rename to src/transformers/data/metrics/__init__.py diff --git a/transformers/data/metrics/squad_metrics.py b/src/transformers/data/metrics/squad_metrics.py similarity index 100% rename from transformers/data/metrics/squad_metrics.py rename to src/transformers/data/metrics/squad_metrics.py diff --git a/transformers/data/processors/__init__.py b/src/transformers/data/processors/__init__.py similarity index 100% rename from transformers/data/processors/__init__.py rename to src/transformers/data/processors/__init__.py diff --git a/transformers/data/processors/glue.py b/src/transformers/data/processors/glue.py similarity index 100% rename from transformers/data/processors/glue.py rename to src/transformers/data/processors/glue.py diff --git a/transformers/data/processors/squad.py b/src/transformers/data/processors/squad.py similarity index 100% rename from transformers/data/processors/squad.py rename to src/transformers/data/processors/squad.py diff --git a/transformers/data/processors/utils.py b/src/transformers/data/processors/utils.py similarity index 100% rename from transformers/data/processors/utils.py rename to src/transformers/data/processors/utils.py diff --git a/transformers/data/processors/xnli.py b/src/transformers/data/processors/xnli.py similarity index 100% rename from transformers/data/processors/xnli.py rename to src/transformers/data/processors/xnli.py diff --git a/transformers/file_utils.py b/src/transformers/file_utils.py similarity index 100% rename from transformers/file_utils.py rename to src/transformers/file_utils.py diff --git a/transformers/hf_api.py b/src/transformers/hf_api.py similarity index 100% rename from transformers/hf_api.py rename to src/transformers/hf_api.py diff --git a/transformers/modelcard.py b/src/transformers/modelcard.py similarity index 100% rename from transformers/modelcard.py rename to src/transformers/modelcard.py diff --git a/transformers/modeling_albert.py b/src/transformers/modeling_albert.py similarity index 100% rename from transformers/modeling_albert.py rename to src/transformers/modeling_albert.py diff --git a/transformers/modeling_auto.py b/src/transformers/modeling_auto.py similarity index 100% rename from transformers/modeling_auto.py rename to src/transformers/modeling_auto.py diff --git a/transformers/modeling_bert.py b/src/transformers/modeling_bert.py similarity index 100% rename from transformers/modeling_bert.py rename to src/transformers/modeling_bert.py diff --git a/transformers/modeling_camembert.py b/src/transformers/modeling_camembert.py similarity index 100% rename from transformers/modeling_camembert.py rename to src/transformers/modeling_camembert.py diff --git a/transformers/modeling_ctrl.py b/src/transformers/modeling_ctrl.py similarity index 100% rename from transformers/modeling_ctrl.py rename to src/transformers/modeling_ctrl.py diff --git a/transformers/modeling_distilbert.py b/src/transformers/modeling_distilbert.py similarity index 100% rename from transformers/modeling_distilbert.py rename to src/transformers/modeling_distilbert.py diff --git a/transformers/modeling_encoder_decoder.py b/src/transformers/modeling_encoder_decoder.py similarity index 100% rename from transformers/modeling_encoder_decoder.py rename to src/transformers/modeling_encoder_decoder.py diff --git a/transformers/modeling_gpt2.py b/src/transformers/modeling_gpt2.py similarity index 100% rename from transformers/modeling_gpt2.py rename to src/transformers/modeling_gpt2.py diff --git a/transformers/modeling_mmbt.py b/src/transformers/modeling_mmbt.py similarity index 100% rename from transformers/modeling_mmbt.py rename to src/transformers/modeling_mmbt.py diff --git a/transformers/modeling_openai.py b/src/transformers/modeling_openai.py similarity index 100% rename from transformers/modeling_openai.py rename to src/transformers/modeling_openai.py diff --git a/transformers/modeling_roberta.py b/src/transformers/modeling_roberta.py similarity index 100% rename from transformers/modeling_roberta.py rename to src/transformers/modeling_roberta.py diff --git a/transformers/modeling_t5.py b/src/transformers/modeling_t5.py similarity index 100% rename from transformers/modeling_t5.py rename to src/transformers/modeling_t5.py diff --git a/transformers/modeling_tf_albert.py b/src/transformers/modeling_tf_albert.py similarity index 100% rename from transformers/modeling_tf_albert.py rename to src/transformers/modeling_tf_albert.py diff --git a/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py similarity index 100% rename from transformers/modeling_tf_auto.py rename to src/transformers/modeling_tf_auto.py diff --git a/transformers/modeling_tf_bert.py b/src/transformers/modeling_tf_bert.py similarity index 100% rename from transformers/modeling_tf_bert.py rename to src/transformers/modeling_tf_bert.py diff --git a/transformers/modeling_tf_ctrl.py b/src/transformers/modeling_tf_ctrl.py similarity index 100% rename from transformers/modeling_tf_ctrl.py rename to src/transformers/modeling_tf_ctrl.py diff --git a/transformers/modeling_tf_distilbert.py b/src/transformers/modeling_tf_distilbert.py similarity index 100% rename from transformers/modeling_tf_distilbert.py rename to src/transformers/modeling_tf_distilbert.py diff --git a/transformers/modeling_tf_gpt2.py b/src/transformers/modeling_tf_gpt2.py similarity index 100% rename from transformers/modeling_tf_gpt2.py rename to src/transformers/modeling_tf_gpt2.py diff --git a/transformers/modeling_tf_openai.py b/src/transformers/modeling_tf_openai.py similarity index 100% rename from transformers/modeling_tf_openai.py rename to src/transformers/modeling_tf_openai.py diff --git a/transformers/modeling_tf_pytorch_utils.py b/src/transformers/modeling_tf_pytorch_utils.py similarity index 100% rename from transformers/modeling_tf_pytorch_utils.py rename to src/transformers/modeling_tf_pytorch_utils.py diff --git a/transformers/modeling_tf_roberta.py b/src/transformers/modeling_tf_roberta.py similarity index 100% rename from transformers/modeling_tf_roberta.py rename to src/transformers/modeling_tf_roberta.py diff --git a/transformers/modeling_tf_t5.py b/src/transformers/modeling_tf_t5.py similarity index 100% rename from transformers/modeling_tf_t5.py rename to src/transformers/modeling_tf_t5.py diff --git a/transformers/modeling_tf_transfo_xl.py b/src/transformers/modeling_tf_transfo_xl.py similarity index 100% rename from transformers/modeling_tf_transfo_xl.py rename to src/transformers/modeling_tf_transfo_xl.py diff --git a/transformers/modeling_tf_transfo_xl_utilities.py b/src/transformers/modeling_tf_transfo_xl_utilities.py similarity index 100% rename from transformers/modeling_tf_transfo_xl_utilities.py rename to src/transformers/modeling_tf_transfo_xl_utilities.py diff --git a/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py similarity index 100% rename from transformers/modeling_tf_utils.py rename to src/transformers/modeling_tf_utils.py diff --git a/transformers/modeling_tf_xlm.py b/src/transformers/modeling_tf_xlm.py similarity index 100% rename from transformers/modeling_tf_xlm.py rename to src/transformers/modeling_tf_xlm.py diff --git a/transformers/modeling_tf_xlnet.py b/src/transformers/modeling_tf_xlnet.py similarity index 100% rename from transformers/modeling_tf_xlnet.py rename to src/transformers/modeling_tf_xlnet.py diff --git a/transformers/modeling_transfo_xl.py b/src/transformers/modeling_transfo_xl.py similarity index 100% rename from transformers/modeling_transfo_xl.py rename to src/transformers/modeling_transfo_xl.py diff --git a/transformers/modeling_transfo_xl_utilities.py b/src/transformers/modeling_transfo_xl_utilities.py similarity index 100% rename from transformers/modeling_transfo_xl_utilities.py rename to src/transformers/modeling_transfo_xl_utilities.py diff --git a/transformers/modeling_utils.py b/src/transformers/modeling_utils.py similarity index 100% rename from transformers/modeling_utils.py rename to src/transformers/modeling_utils.py diff --git a/transformers/modeling_xlm.py b/src/transformers/modeling_xlm.py similarity index 100% rename from transformers/modeling_xlm.py rename to src/transformers/modeling_xlm.py diff --git a/transformers/modeling_xlm_roberta.py b/src/transformers/modeling_xlm_roberta.py similarity index 100% rename from transformers/modeling_xlm_roberta.py rename to src/transformers/modeling_xlm_roberta.py diff --git a/transformers/modeling_xlnet.py b/src/transformers/modeling_xlnet.py similarity index 100% rename from transformers/modeling_xlnet.py rename to src/transformers/modeling_xlnet.py diff --git a/transformers/optimization.py b/src/transformers/optimization.py similarity index 100% rename from transformers/optimization.py rename to src/transformers/optimization.py diff --git a/transformers/optimization_tf.py b/src/transformers/optimization_tf.py similarity index 100% rename from transformers/optimization_tf.py rename to src/transformers/optimization_tf.py diff --git a/transformers/pipelines.py b/src/transformers/pipelines.py similarity index 100% rename from transformers/pipelines.py rename to src/transformers/pipelines.py diff --git a/transformers/tokenization_albert.py b/src/transformers/tokenization_albert.py similarity index 100% rename from transformers/tokenization_albert.py rename to src/transformers/tokenization_albert.py diff --git a/transformers/tokenization_auto.py b/src/transformers/tokenization_auto.py similarity index 100% rename from transformers/tokenization_auto.py rename to src/transformers/tokenization_auto.py diff --git a/transformers/tokenization_bert.py b/src/transformers/tokenization_bert.py similarity index 100% rename from transformers/tokenization_bert.py rename to src/transformers/tokenization_bert.py diff --git a/transformers/tokenization_bert_japanese.py b/src/transformers/tokenization_bert_japanese.py similarity index 100% rename from transformers/tokenization_bert_japanese.py rename to src/transformers/tokenization_bert_japanese.py diff --git a/transformers/tokenization_camembert.py b/src/transformers/tokenization_camembert.py similarity index 100% rename from transformers/tokenization_camembert.py rename to src/transformers/tokenization_camembert.py diff --git a/transformers/tokenization_ctrl.py b/src/transformers/tokenization_ctrl.py similarity index 100% rename from transformers/tokenization_ctrl.py rename to src/transformers/tokenization_ctrl.py diff --git a/transformers/tokenization_distilbert.py b/src/transformers/tokenization_distilbert.py similarity index 100% rename from transformers/tokenization_distilbert.py rename to src/transformers/tokenization_distilbert.py diff --git a/transformers/tokenization_gpt2.py b/src/transformers/tokenization_gpt2.py similarity index 100% rename from transformers/tokenization_gpt2.py rename to src/transformers/tokenization_gpt2.py diff --git a/transformers/tokenization_openai.py b/src/transformers/tokenization_openai.py similarity index 100% rename from transformers/tokenization_openai.py rename to src/transformers/tokenization_openai.py diff --git a/transformers/tokenization_roberta.py b/src/transformers/tokenization_roberta.py similarity index 100% rename from transformers/tokenization_roberta.py rename to src/transformers/tokenization_roberta.py diff --git a/transformers/tokenization_t5.py b/src/transformers/tokenization_t5.py similarity index 100% rename from transformers/tokenization_t5.py rename to src/transformers/tokenization_t5.py diff --git a/transformers/tokenization_transfo_xl.py b/src/transformers/tokenization_transfo_xl.py similarity index 100% rename from transformers/tokenization_transfo_xl.py rename to src/transformers/tokenization_transfo_xl.py diff --git a/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py similarity index 100% rename from transformers/tokenization_utils.py rename to src/transformers/tokenization_utils.py diff --git a/transformers/tokenization_xlm.py b/src/transformers/tokenization_xlm.py similarity index 100% rename from transformers/tokenization_xlm.py rename to src/transformers/tokenization_xlm.py diff --git a/transformers/tokenization_xlm_roberta.py b/src/transformers/tokenization_xlm_roberta.py similarity index 100% rename from transformers/tokenization_xlm_roberta.py rename to src/transformers/tokenization_xlm_roberta.py diff --git a/transformers/tokenization_xlnet.py b/src/transformers/tokenization_xlnet.py similarity index 100% rename from transformers/tokenization_xlnet.py rename to src/transformers/tokenization_xlnet.py