From d12fc9d3fe7508efaa740fb98bcd0caf18033a88 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Tue, 7 Apr 2026 13:50:02 -0700 Subject: [PATCH] Update license headers from Llama 2/3 to Llama The copyright headers referenced Llama 2 or Llama 3 Community License Agreement but the code supports all Llama versions (2 through 4). Update to the generic form. Also update check_copyright_header.py to enforce the new format. --- .github/scripts/check_copyright_header.py | 4 ++-- .github/scripts/spellcheck.sh | 2 +- 3p-integrations/tgi/merge_lora_weights.py | 2 +- 3p-integrations/vllm/inference.py | 2 +- end-to-end-use-cases/RAFT-Chatbot/config.py | 2 +- end-to-end-use-cases/RAFT-Chatbot/raft_eval.py | 2 +- end-to-end-use-cases/RAFT-Chatbot/raft_utils.py | 2 +- .../inference/cloud/azure/chat_azure_api_benchmark.py | 2 +- .../cloud/azure/pretrained_azure_api_benchmark.py | 2 +- .../inference/on_prem/vllm/chat_vllm_benchmark.py | 2 +- .../inference/on_prem/vllm/pretrained_vllm_benchmark.py | 2 +- .../llm_eval_harness/meta_eval/prepare_meta_eval.py | 2 +- end-to-end-use-cases/coding/text2sql/csv2db.py | 2 +- end-to-end-use-cases/coding/text2sql/txt2csv.py | 2 +- getting-started/finetuning/datasets/custom_dataset.py | 2 +- getting-started/finetuning/datasets/ocrvqa_dataset.py | 2 +- getting-started/finetuning/datasets/raft_dataset.py | 2 +- getting-started/finetuning/finetuning.py | 2 +- .../finetuning/quickstart_peft_finetuning.ipynb | 2 +- .../local_inference/chat_completion/chat_completion.py | 2 +- getting-started/inference/local_inference/inference.py | 2 +- src/llama_cookbook/configs/__init__.py | 2 +- src/llama_cookbook/configs/datasets.py | 2 +- src/llama_cookbook/configs/fsdp.py | 2 +- src/llama_cookbook/configs/peft.py | 2 +- src/llama_cookbook/configs/quantization.py | 2 +- src/llama_cookbook/configs/training.py | 2 +- src/llama_cookbook/configs/wandb.py | 2 +- src/llama_cookbook/data/__init__.py | 2 +- src/llama_cookbook/data/concatenator.py | 2 +- .../llama_guard/finetuning_data_formatter_example.py | 2 +- src/llama_cookbook/data/sampler.py | 2 +- src/llama_cookbook/datasets/__init__.py | 2 +- src/llama_cookbook/datasets/alpaca_dataset.py | 2 +- src/llama_cookbook/datasets/grammar_dataset/__init__.py | 2 +- .../datasets/grammar_dataset/grammar_dataset.py | 2 +- .../grammar_dataset/grammar_dataset_process.ipynb | 2 +- src/llama_cookbook/datasets/samsum_dataset.py | 2 +- src/llama_cookbook/finetuning.py | 2 +- src/llama_cookbook/inference/__init__.py | 2 +- src/llama_cookbook/inference/chat_utils.py | 2 +- .../inference/checkpoint_converter_fsdp_hf.py | 2 +- src/llama_cookbook/inference/prompt_format_utils.py | 2 +- src/llama_cookbook/inference/safety_utils.py | 2 +- src/llama_cookbook/model_checkpointing/__init__.py | 2 +- .../model_checkpointing/checkpoint_handler.py | 2 +- src/llama_cookbook/policies/__init__.py | 2 +- .../policies/activation_checkpointing_functions.py | 2 +- src/llama_cookbook/policies/anyprecision_optimizer.py | 2 +- src/llama_cookbook/policies/mixed_precision.py | 2 +- src/llama_cookbook/policies/wrapping.py | 2 +- src/llama_cookbook/tools/compare_llama_weights.py | 2 +- src/llama_cookbook/tools/convert_hf_weights_to_llama.py | 2 +- src/llama_cookbook/utils/__init__.py | 2 +- src/llama_cookbook/utils/config_utils.py | 2 +- src/llama_cookbook/utils/dataset_utils.py | 2 +- src/llama_cookbook/utils/fsdp_utils.py | 2 +- src/llama_cookbook/utils/memory_utils.py | 2 +- src/llama_cookbook/utils/plot_metrics.py | 2 +- src/llama_cookbook/utils/train_utils.py | 9 +++++++-- src/tests/conftest.py | 2 +- src/tests/datasets/test_custom_dataset.py | 2 +- src/tests/datasets/test_grammar_datasets.py | 2 +- src/tests/datasets/test_samsum_datasets.py | 2 +- src/tests/test_batching.py | 2 +- src/tests/test_finetuning.py | 2 +- src/tests/test_sampler.py | 2 +- src/tests/test_train_utils.py | 2 +- src/tests/utils.py | 2 +- 69 files changed, 76 insertions(+), 71 deletions(-) diff --git a/.github/scripts/check_copyright_header.py b/.github/scripts/check_copyright_header.py index 6d5f2eaef..4777a71a7 100644 --- a/.github/scripts/check_copyright_header.py +++ b/.github/scripts/check_copyright_header.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import re from pathlib import Path @@ -8,7 +8,7 @@ PATTERN = "(Meta Platforms, Inc. and affiliates)|(Facebook, Inc(\.|,)? and its affiliates)|([0-9]{4}-present(\.|,)? Facebook)|([0-9]{4}(\.|,)? Facebook)" HEADER = """# Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement.\n\n""" +# This software may be used and distributed according to the terms of the Llama Community License Agreement.\n\n""" #Files in black list must be relative to main repo folder BLACKLIST = ["tools/benchmarks/llm_eval_harness/open_llm_leaderboard/hellaswag_utils.py"] diff --git a/.github/scripts/spellcheck.sh b/.github/scripts/spellcheck.sh index 4d3f1f44c..4854973b7 100755 --- a/.github/scripts/spellcheck.sh +++ b/.github/scripts/spellcheck.sh @@ -1,6 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # Source: https://github.com/pytorch/torchx/blob/main/scripts/spellcheck.sh set -ex sudo apt-get install aspell diff --git a/3p-integrations/tgi/merge_lora_weights.py b/3p-integrations/tgi/merge_lora_weights.py index b5137f1c6..742da91cd 100644 --- a/3p-integrations/tgi/merge_lora_weights.py +++ b/3p-integrations/tgi/merge_lora_weights.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import fire import torch diff --git a/3p-integrations/vllm/inference.py b/3p-integrations/vllm/inference.py index c56d8d2cb..6f742ad04 100644 --- a/3p-integrations/vllm/inference.py +++ b/3p-integrations/vllm/inference.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import uuid import asyncio diff --git a/end-to-end-use-cases/RAFT-Chatbot/config.py b/end-to-end-use-cases/RAFT-Chatbot/config.py index 60152f40f..6ed3487ac 100644 --- a/end-to-end-use-cases/RAFT-Chatbot/config.py +++ b/end-to-end-use-cases/RAFT-Chatbot/config.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import yaml diff --git a/end-to-end-use-cases/RAFT-Chatbot/raft_eval.py b/end-to-end-use-cases/RAFT-Chatbot/raft_eval.py index 73c1361db..911e67b59 100644 --- a/end-to-end-use-cases/RAFT-Chatbot/raft_eval.py +++ b/end-to-end-use-cases/RAFT-Chatbot/raft_eval.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import logging import evaluate import argparse diff --git a/end-to-end-use-cases/RAFT-Chatbot/raft_utils.py b/end-to-end-use-cases/RAFT-Chatbot/raft_utils.py index 875f77714..9fe25f7ec 100644 --- a/end-to-end-use-cases/RAFT-Chatbot/raft_utils.py +++ b/end-to-end-use-cases/RAFT-Chatbot/raft_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import os import logging diff --git a/end-to-end-use-cases/benchmarks/inference/cloud/azure/chat_azure_api_benchmark.py b/end-to-end-use-cases/benchmarks/inference/cloud/azure/chat_azure_api_benchmark.py index bb653131b..741155f6f 100644 --- a/end-to-end-use-cases/benchmarks/inference/cloud/azure/chat_azure_api_benchmark.py +++ b/end-to-end-use-cases/benchmarks/inference/cloud/azure/chat_azure_api_benchmark.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import csv import json diff --git a/end-to-end-use-cases/benchmarks/inference/cloud/azure/pretrained_azure_api_benchmark.py b/end-to-end-use-cases/benchmarks/inference/cloud/azure/pretrained_azure_api_benchmark.py index 4d66912c4..8881fac03 100644 --- a/end-to-end-use-cases/benchmarks/inference/cloud/azure/pretrained_azure_api_benchmark.py +++ b/end-to-end-use-cases/benchmarks/inference/cloud/azure/pretrained_azure_api_benchmark.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import csv import json diff --git a/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/chat_vllm_benchmark.py b/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/chat_vllm_benchmark.py index 167c84a08..f2a5277b2 100644 --- a/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/chat_vllm_benchmark.py +++ b/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/chat_vllm_benchmark.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import csv import json diff --git a/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/pretrained_vllm_benchmark.py b/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/pretrained_vllm_benchmark.py index fd1594a31..ea44403d9 100644 --- a/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/pretrained_vllm_benchmark.py +++ b/end-to-end-use-cases/benchmarks/inference/on_prem/vllm/pretrained_vllm_benchmark.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import csv import json diff --git a/end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/prepare_meta_eval.py b/end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/prepare_meta_eval.py index 8d500d481..0e6676cde 100644 --- a/end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/prepare_meta_eval.py +++ b/end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/prepare_meta_eval.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import argparse import errno diff --git a/end-to-end-use-cases/coding/text2sql/csv2db.py b/end-to-end-use-cases/coding/text2sql/csv2db.py index 44ad0ef0a..6406a9d7a 100644 --- a/end-to-end-use-cases/coding/text2sql/csv2db.py +++ b/end-to-end-use-cases/coding/text2sql/csv2db.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import sqlite3 import csv diff --git a/end-to-end-use-cases/coding/text2sql/txt2csv.py b/end-to-end-use-cases/coding/text2sql/txt2csv.py index 4470d2047..30ec8b392 100644 --- a/end-to-end-use-cases/coding/text2sql/txt2csv.py +++ b/end-to-end-use-cases/coding/text2sql/txt2csv.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import csv diff --git a/getting-started/finetuning/datasets/custom_dataset.py b/getting-started/finetuning/datasets/custom_dataset.py index 848d4223e..9be4bc2aa 100644 --- a/getting-started/finetuning/datasets/custom_dataset.py +++ b/getting-started/finetuning/datasets/custom_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # For dataset details visit: https://huggingface.co/datasets/samsum diff --git a/getting-started/finetuning/datasets/ocrvqa_dataset.py b/getting-started/finetuning/datasets/ocrvqa_dataset.py index 9597cac62..19671d127 100644 --- a/getting-started/finetuning/datasets/ocrvqa_dataset.py +++ b/getting-started/finetuning/datasets/ocrvqa_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import copy diff --git a/getting-started/finetuning/datasets/raft_dataset.py b/getting-started/finetuning/datasets/raft_dataset.py index 9341dd317..113b4b2e7 100644 --- a/getting-started/finetuning/datasets/raft_dataset.py +++ b/getting-started/finetuning/datasets/raft_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import copy diff --git a/getting-started/finetuning/finetuning.py b/getting-started/finetuning/finetuning.py index d34579da8..9ef8a5380 100644 --- a/getting-started/finetuning/finetuning.py +++ b/getting-started/finetuning/finetuning.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import fire from llama_cookbook.finetuning import main diff --git a/getting-started/finetuning/quickstart_peft_finetuning.ipynb b/getting-started/finetuning/quickstart_peft_finetuning.ipynb index 79096fee3..a6ad5fe24 100644 --- a/getting-started/finetuning/quickstart_peft_finetuning.ipynb +++ b/getting-started/finetuning/quickstart_peft_finetuning.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "source": [ "Copyright (c) Meta Platforms, Inc. and affiliates.\n", - "This software may be used and distributed according to the terms of the Llama 3 Community License Agreement.\n", + "This software may be used and distributed according to the terms of the Llama Community License Agreement.\n", "\n", "\"Open" ] diff --git a/getting-started/inference/local_inference/chat_completion/chat_completion.py b/getting-started/inference/local_inference/chat_completion/chat_completion.py index b627083b7..8a40b165e 100644 --- a/getting-started/inference/local_inference/chat_completion/chat_completion.py +++ b/getting-started/inference/local_inference/chat_completion/chat_completion.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # from accelerate import init_empty_weights, load_checkpoint_and_dispatch diff --git a/getting-started/inference/local_inference/inference.py b/getting-started/inference/local_inference/inference.py index 0323a1c9d..e7fb923cf 100644 --- a/getting-started/inference/local_inference/inference.py +++ b/getting-started/inference/local_inference/inference.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 3 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import os import sys diff --git a/src/llama_cookbook/configs/__init__.py b/src/llama_cookbook/configs/__init__.py index dabe7e331..84d6d12e9 100644 --- a/src/llama_cookbook/configs/__init__.py +++ b/src/llama_cookbook/configs/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from llama_cookbook.configs.peft import lora_config, llama_adapter_config, prefix_config from llama_cookbook.configs.fsdp import fsdp_config diff --git a/src/llama_cookbook/configs/datasets.py b/src/llama_cookbook/configs/datasets.py index ff84b331a..6135925b6 100644 --- a/src/llama_cookbook/configs/datasets.py +++ b/src/llama_cookbook/configs/datasets.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass diff --git a/src/llama_cookbook/configs/fsdp.py b/src/llama_cookbook/configs/fsdp.py index 0fedfffea..87ac88ef4 100644 --- a/src/llama_cookbook/configs/fsdp.py +++ b/src/llama_cookbook/configs/fsdp.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass diff --git a/src/llama_cookbook/configs/peft.py b/src/llama_cookbook/configs/peft.py index 7140e025d..603bc6b28 100644 --- a/src/llama_cookbook/configs/peft.py +++ b/src/llama_cookbook/configs/peft.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass, field from typing import List diff --git a/src/llama_cookbook/configs/quantization.py b/src/llama_cookbook/configs/quantization.py index ecefa2faf..aaf475720 100644 --- a/src/llama_cookbook/configs/quantization.py +++ b/src/llama_cookbook/configs/quantization.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass from typing import Optional diff --git a/src/llama_cookbook/configs/training.py b/src/llama_cookbook/configs/training.py index 75f80790c..aafc996ed 100644 --- a/src/llama_cookbook/configs/training.py +++ b/src/llama_cookbook/configs/training.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass diff --git a/src/llama_cookbook/configs/wandb.py b/src/llama_cookbook/configs/wandb.py index 129760580..6f96e0488 100644 --- a/src/llama_cookbook/configs/wandb.py +++ b/src/llama_cookbook/configs/wandb.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from typing import List, Optional from dataclasses import dataclass, field diff --git a/src/llama_cookbook/data/__init__.py b/src/llama_cookbook/data/__init__.py index 54ed04d3b..2ed8f0e43 100644 --- a/src/llama_cookbook/data/__init__.py +++ b/src/llama_cookbook/data/__init__.py @@ -1,2 +1,2 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. \ No newline at end of file +# This software may be used and distributed according to the terms of the Llama Community License Agreement. \ No newline at end of file diff --git a/src/llama_cookbook/data/concatenator.py b/src/llama_cookbook/data/concatenator.py index da50322c3..013ad3ced 100644 --- a/src/llama_cookbook/data/concatenator.py +++ b/src/llama_cookbook/data/concatenator.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from tqdm import tqdm from itertools import chain diff --git a/src/llama_cookbook/data/llama_guard/finetuning_data_formatter_example.py b/src/llama_cookbook/data/llama_guard/finetuning_data_formatter_example.py index 3493cf180..6aed131a8 100644 --- a/src/llama_cookbook/data/llama_guard/finetuning_data_formatter_example.py +++ b/src/llama_cookbook/data/llama_guard/finetuning_data_formatter_example.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from finetuning_data_formatter import TrainingExample, Guidelines, Category, LlamaGuardPromptConfigs, LlamaGuardGenerationConfigs, ExplanationPosition, AugmentationConfigs, FormatterConfigs, create_formatted_finetuning_examples diff --git a/src/llama_cookbook/data/sampler.py b/src/llama_cookbook/data/sampler.py index 8798b641e..bd7101ac5 100644 --- a/src/llama_cookbook/data/sampler.py +++ b/src/llama_cookbook/data/sampler.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import random from itertools import islice diff --git a/src/llama_cookbook/datasets/__init__.py b/src/llama_cookbook/datasets/__init__.py index a718524c9..f47ce0732 100644 --- a/src/llama_cookbook/datasets/__init__.py +++ b/src/llama_cookbook/datasets/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from functools import partial diff --git a/src/llama_cookbook/datasets/alpaca_dataset.py b/src/llama_cookbook/datasets/alpaca_dataset.py index 396551d1b..a71fd6ff8 100644 --- a/src/llama_cookbook/datasets/alpaca_dataset.py +++ b/src/llama_cookbook/datasets/alpaca_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # For dataset details visit: https://crfm.stanford.edu/2023/03/13/alpaca.html diff --git a/src/llama_cookbook/datasets/grammar_dataset/__init__.py b/src/llama_cookbook/datasets/grammar_dataset/__init__.py index b193f6796..5d64cd649 100644 --- a/src/llama_cookbook/datasets/grammar_dataset/__init__.py +++ b/src/llama_cookbook/datasets/grammar_dataset/__init__.py @@ -1,3 +1,3 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. diff --git a/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset.py b/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset.py index efd54f230..81492d1a3 100644 --- a/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset.py +++ b/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # For dataset details visit: https://huggingface.co/datasets/jfleg # For download and preparation see: recipes/ft_datasets/grammar_dataset/grammar_dataset_process.ipynb diff --git a/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset_process.ipynb b/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset_process.ipynb index 2637a3cbd..260e88bb9 100644 --- a/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset_process.ipynb +++ b/src/llama_cookbook/datasets/grammar_dataset/grammar_dataset_process.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "source": [ "Copyright (c) Meta Platforms, Inc. and affiliates.\n", - "This software may be used and distributed according to the terms of the Llama 2 Community License Agreement.\n", + "This software may be used and distributed according to the terms of the Llama Community License Agreement.\n", "\n", "Use this notebook to pull in datasets and apply pre-processing. Most grammar datasets unfortunately require preprocessing before being usable in training. (example - jfleg has 4 targets per input, so we have to rematch as 1:1 pairings) " ] diff --git a/src/llama_cookbook/datasets/samsum_dataset.py b/src/llama_cookbook/datasets/samsum_dataset.py index 938377cb6..d9683dead 100644 --- a/src/llama_cookbook/datasets/samsum_dataset.py +++ b/src/llama_cookbook/datasets/samsum_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # For dataset details visit: https://huggingface.co/datasets/samsum diff --git a/src/llama_cookbook/finetuning.py b/src/llama_cookbook/finetuning.py index 05d5babc9..f19763913 100644 --- a/src/llama_cookbook/finetuning.py +++ b/src/llama_cookbook/finetuning.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import dataclasses import os diff --git a/src/llama_cookbook/inference/__init__.py b/src/llama_cookbook/inference/__init__.py index 54ed04d3b..2ed8f0e43 100644 --- a/src/llama_cookbook/inference/__init__.py +++ b/src/llama_cookbook/inference/__init__.py @@ -1,2 +1,2 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. \ No newline at end of file +# This software may be used and distributed according to the terms of the Llama Community License Agreement. \ No newline at end of file diff --git a/src/llama_cookbook/inference/chat_utils.py b/src/llama_cookbook/inference/chat_utils.py index 06493ee94..3ee0321a6 100644 --- a/src/llama_cookbook/inference/chat_utils.py +++ b/src/llama_cookbook/inference/chat_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import json diff --git a/src/llama_cookbook/inference/checkpoint_converter_fsdp_hf.py b/src/llama_cookbook/inference/checkpoint_converter_fsdp_hf.py index 000fa0ffb..201a00249 100644 --- a/src/llama_cookbook/inference/checkpoint_converter_fsdp_hf.py +++ b/src/llama_cookbook/inference/checkpoint_converter_fsdp_hf.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # from accelerate import init_empty_weights, load_checkpoint_and_dispatch diff --git a/src/llama_cookbook/inference/prompt_format_utils.py b/src/llama_cookbook/inference/prompt_format_utils.py index a791c89d6..e9ec69ae8 100644 --- a/src/llama_cookbook/inference/prompt_format_utils.py +++ b/src/llama_cookbook/inference/prompt_format_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass from string import Template diff --git a/src/llama_cookbook/inference/safety_utils.py b/src/llama_cookbook/inference/safety_utils.py index b22159afd..cd13b724e 100644 --- a/src/llama_cookbook/inference/safety_utils.py +++ b/src/llama_cookbook/inference/safety_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import os import torch diff --git a/src/llama_cookbook/model_checkpointing/__init__.py b/src/llama_cookbook/model_checkpointing/__init__.py index 23cc54abf..9ecc70a6a 100644 --- a/src/llama_cookbook/model_checkpointing/__init__.py +++ b/src/llama_cookbook/model_checkpointing/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from llama_cookbook.model_checkpointing.checkpoint_handler import ( load_model_checkpoint, diff --git a/src/llama_cookbook/model_checkpointing/checkpoint_handler.py b/src/llama_cookbook/model_checkpointing/checkpoint_handler.py index 933c28908..558ad2387 100644 --- a/src/llama_cookbook/model_checkpointing/checkpoint_handler.py +++ b/src/llama_cookbook/model_checkpointing/checkpoint_handler.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from pathlib import Path from datetime import datetime diff --git a/src/llama_cookbook/policies/__init__.py b/src/llama_cookbook/policies/__init__.py index 1017bae34..8208f4ce7 100644 --- a/src/llama_cookbook/policies/__init__.py +++ b/src/llama_cookbook/policies/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from llama_cookbook.policies.mixed_precision import * from llama_cookbook.policies.wrapping import * diff --git a/src/llama_cookbook/policies/activation_checkpointing_functions.py b/src/llama_cookbook/policies/activation_checkpointing_functions.py index 818b7dace..9951813bd 100644 --- a/src/llama_cookbook/policies/activation_checkpointing_functions.py +++ b/src/llama_cookbook/policies/activation_checkpointing_functions.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from functools import partial diff --git a/src/llama_cookbook/policies/anyprecision_optimizer.py b/src/llama_cookbook/policies/anyprecision_optimizer.py index f110ed6fc..eb6be092c 100644 --- a/src/llama_cookbook/policies/anyprecision_optimizer.py +++ b/src/llama_cookbook/policies/anyprecision_optimizer.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. # AnyPrecisionAdamW: a flexible precision AdamW optimizer # with optional Kahan summation for high precision weight updates. diff --git a/src/llama_cookbook/policies/mixed_precision.py b/src/llama_cookbook/policies/mixed_precision.py index 11df7edf6..fec3d7dff 100644 --- a/src/llama_cookbook/policies/mixed_precision.py +++ b/src/llama_cookbook/policies/mixed_precision.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import torch diff --git a/src/llama_cookbook/policies/wrapping.py b/src/llama_cookbook/policies/wrapping.py index 6d67b940e..64bae0eb0 100644 --- a/src/llama_cookbook/policies/wrapping.py +++ b/src/llama_cookbook/policies/wrapping.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import functools diff --git a/src/llama_cookbook/tools/compare_llama_weights.py b/src/llama_cookbook/tools/compare_llama_weights.py index 25d16aa76..dbbf48057 100644 --- a/src/llama_cookbook/tools/compare_llama_weights.py +++ b/src/llama_cookbook/tools/compare_llama_weights.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import gc import glob diff --git a/src/llama_cookbook/tools/convert_hf_weights_to_llama.py b/src/llama_cookbook/tools/convert_hf_weights_to_llama.py index 356e4a4b9..842d0bf2f 100644 --- a/src/llama_cookbook/tools/convert_hf_weights_to_llama.py +++ b/src/llama_cookbook/tools/convert_hf_weights_to_llama.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import json import os diff --git a/src/llama_cookbook/utils/__init__.py b/src/llama_cookbook/utils/__init__.py index dc8c427ad..19a470d27 100644 --- a/src/llama_cookbook/utils/__init__.py +++ b/src/llama_cookbook/utils/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from llama_cookbook.utils.memory_utils import MemoryTrace from llama_cookbook.utils.dataset_utils import * diff --git a/src/llama_cookbook/utils/config_utils.py b/src/llama_cookbook/utils/config_utils.py index eb4510bb7..be9d84447 100644 --- a/src/llama_cookbook/utils/config_utils.py +++ b/src/llama_cookbook/utils/config_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import inspect from dataclasses import asdict diff --git a/src/llama_cookbook/utils/dataset_utils.py b/src/llama_cookbook/utils/dataset_utils.py index 77a3c5db0..0cc9126d0 100644 --- a/src/llama_cookbook/utils/dataset_utils.py +++ b/src/llama_cookbook/utils/dataset_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import torch diff --git a/src/llama_cookbook/utils/fsdp_utils.py b/src/llama_cookbook/utils/fsdp_utils.py index 3530c60d4..571e73fd6 100644 --- a/src/llama_cookbook/utils/fsdp_utils.py +++ b/src/llama_cookbook/utils/fsdp_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import os import torch import torch.cuda.nccl as nccl diff --git a/src/llama_cookbook/utils/memory_utils.py b/src/llama_cookbook/utils/memory_utils.py index 3fa06e1ac..325c3a26c 100644 --- a/src/llama_cookbook/utils/memory_utils.py +++ b/src/llama_cookbook/utils/memory_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import gc import psutil diff --git a/src/llama_cookbook/utils/plot_metrics.py b/src/llama_cookbook/utils/plot_metrics.py index e8ab2305c..d61330142 100644 --- a/src/llama_cookbook/utils/plot_metrics.py +++ b/src/llama_cookbook/utils/plot_metrics.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import json import matplotlib.pyplot as plt diff --git a/src/llama_cookbook/utils/train_utils.py b/src/llama_cookbook/utils/train_utils.py index b6858422a..bcaeec10a 100644 --- a/src/llama_cookbook/utils/train_utils.py +++ b/src/llama_cookbook/utils/train_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import os import time @@ -22,7 +22,12 @@ from llama_cookbook.model_checkpointing import save_fsdp_model_checkpoint_full, save_model_and_optimizer_sharded, save_optimizer_checkpoint, save_peft_checkpoint, save_model_checkpoint from llama_cookbook.policies import fpSixteen,bfSixteen, get_llama_wrapper from llama_cookbook.utils.memory_utils import MemoryTrace -from accelerate.utils import is_xpu_available, is_ccl_available +from accelerate.utils import is_xpu_available +try: + from accelerate.utils import is_ccl_available +except ImportError: + def is_ccl_available(): + return False from llama_cookbook.utils.flop_utils import FlopMeasure def set_tokenizer_params(tokenizer: LlamaTokenizer): tokenizer.pad_token_id = 0 diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 1476bf3c1..0070bf2bc 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import pytest diff --git a/src/tests/datasets/test_custom_dataset.py b/src/tests/datasets/test_custom_dataset.py index 9038dfad3..fd564c196 100644 --- a/src/tests/datasets/test_custom_dataset.py +++ b/src/tests/datasets/test_custom_dataset.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import pytest from contextlib import nullcontext diff --git a/src/tests/datasets/test_grammar_datasets.py b/src/tests/datasets/test_grammar_datasets.py index 29924e0f7..ae39de7f8 100644 --- a/src/tests/datasets/test_grammar_datasets.py +++ b/src/tests/datasets/test_grammar_datasets.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from pathlib import Path import pytest diff --git a/src/tests/datasets/test_samsum_datasets.py b/src/tests/datasets/test_samsum_datasets.py index 7ae75759d..c8efad04a 100644 --- a/src/tests/datasets/test_samsum_datasets.py +++ b/src/tests/datasets/test_samsum_datasets.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from dataclasses import dataclass from functools import partial diff --git a/src/tests/test_batching.py b/src/tests/test_batching.py index ff9392eac..6ba5e8896 100644 --- a/src/tests/test_batching.py +++ b/src/tests/test_batching.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import pytest from contextlib import nullcontext diff --git a/src/tests/test_finetuning.py b/src/tests/test_finetuning.py index 10721113a..3d2cfdee0 100644 --- a/src/tests/test_finetuning.py +++ b/src/tests/test_finetuning.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import os from contextlib import nullcontext diff --git a/src/tests/test_sampler.py b/src/tests/test_sampler.py index e77d22ab1..d789b8164 100644 --- a/src/tests/test_sampler.py +++ b/src/tests/test_sampler.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. import random import pytest diff --git a/src/tests/test_train_utils.py b/src/tests/test_train_utils.py index 06ac0eca7..b55e19ce0 100644 --- a/src/tests/test_train_utils.py +++ b/src/tests/test_train_utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from unittest.mock import patch import pytest diff --git a/src/tests/utils.py b/src/tests/utils.py index 14b96a9a1..b90927c87 100644 --- a/src/tests/utils.py +++ b/src/tests/utils.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# This software may be used and distributed according to the terms of the Llama Community License Agreement. from transformers import AutoTokenizer