Add model like (#14992)

* Add new model like command

* Bad doc-styler

* black and doc-styler, stop fighting!

* black and doc-styler, stop fighting!

* At last

* Clean up

* Typo

* Bad doc-styler

* Bad doc-styler

* All good maybe?

* Use constants

* Add doc and type hints

* More cleaning

* Add doc

* Fix Copied from

* Doc template

* Use typing.Pattern instead

* Framework-specific files

* Fixes

* Select frameworks clean model init

* Deal with frameworks in main init

* fixes

* Last fix

* Prompt user for info

* Delete exemple config

* Last fixes

* Add test config

* Fix bug with model_type included in each other

* Fixes

* More fixes

* More fixes

* Adapt config

* Remove print statements

* Will fix tokenization later, leave it broken for now

* Add test

* Quality

* Try this way

* Debug

* Maybe by setting the path?

* Let's try another way

* It should go better when actually passing the arg...

* Remove debug statements and style

* Fix config

* Add tests

* Test require the three backends

* intermediate commit

* Revamp pattern replacements and start work on feature extractors

* Adapt model info

* Finalize code for processors

* Fix in main init additions

* Finish questionnaire for processing classes

* Fix file name

* Fix for real

* Fix patterns

* Style

* Remove needless warnings

* Copied from should work now.

* Include Copied form in blocks

* Add test

* More fixes and tests

* Apply suggestions from code review

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

* Address review comment

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

.github/workflows/add-model-like.yml

@@ -0,0 +1,60 @@
+name: Add model like runner
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - "src/**"
+      - "tests/**"
+      - ".github/**"
+    types: [opened, synchronize, reopened]
+
+jobs:
+  run_tests_templates:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Loading cache.
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: ~/.cache/pip
+          key: v1-tests_model_like
+          restore-keys: |
+            v1-tests_model_like-${{ hashFiles('setup.py') }}
+            v1-tests_model_like
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip!=21.3
+          sudo apt -y update && sudo apt install -y libsndfile1-dev
+          pip install .[dev]
+
+      - name: Create model files
+        run: |
+          transformers-cli add-new-model-like --config_file tests/fixtures/add_distilbert_like_config.json --path_to_repo .
+          make style
+          make fix-copies
+
+      - name: Run all PyTorch modeling test
+        run: |
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_new_models tests/test_modeling_bert_new.py
+
+      - name: Run style changes
+        run: |
+          git fetch origin master:master
+          make style && make quality && make repo-consistency
+
+      - name: Failure short reports
+        if: ${{ always() }}
+        run: cat reports/tests_new_models_failures_short.txt
+
+      - name: Test suite reports artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: run_all_tests_new_models_test_reports
+          path: reports

src/transformers/commands/transformers_cli.py

@@ -16,6 +16,7 @@
 from argparse import ArgumentParser
 
 from .add_new_model import AddNewModelCommand
+from .add_new_model_like import AddNewModelLikeCommand
 from .convert import ConvertCommand
 from .download import DownloadCommand
 from .env import EnvironmentCommand
@@ -37,6 +38,7 @@ def main():
     ServeCommand.register_subcommand(commands_parser)
     UserCommands.register_subcommand(commands_parser)
     AddNewModelCommand.register_subcommand(commands_parser)
+    AddNewModelLikeCommand.register_subcommand(commands_parser)
     LfsCommands.register_subcommand(commands_parser)
 
     # Let's go

tests/fixtures/add_distilbert_like_config.json

@@ -0,0 +1,19 @@
+{
+    "add_copied_from": true,
+    "old_model_type": "distilbert",
+    "new_model_patterns": {
+        "model_name": "BERT New",
+        "checkpoint": "huggingface/bert-new-base",
+        "model_type": "bert-new",
+        "model_lower_cased": "bert_new",
+        "model_camel_cased": "BertNew",
+        "model_upper_cased": "BERT_NEW",
+        "config_class": "BertNewConfig",
+        "tokenizer_class": "DistilBertTokenizer"
+    },
+    "frameworks": [
+        "pt",
+        "tf",
+        "flax"
+    ]
+} 

utils/tests_fetcher.py

@@ -243,6 +243,7 @@ def create_reverse_dependency_map():
 # Any module file that has a test name which can't be inferred automatically from its name should go here. A better
 # approach is to (re-)name the test file accordingly, and second best to add the correspondence map here.
 SPECIAL_MODULE_TO_TEST_MAP = {
+    "commands/add_new_model_like.py": "test_add_new_model_like.py",
     "configuration_utils.py": "test_configuration_common.py",
     "convert_graph_to_onnx.py": "test_onnx.py",
     "data/data_collator.py": "test_data_collator.py",