Support weight-only quantization with quantized operators in intel-extension-for-transformers.

.github/workflows/test_inc.yml

@@ -30,8 +30,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install cmake
+        pip install py-cpuinfo
+        pip install torch==2.1.0 torchaudio==2.1.0 torchvision==0.16 --extra-index-url https://download.pytorch.org/whl/cpu
         pip install .[neural-compressor,diffusers,tests]
-        pip install intel-extension-for-pytorch
+        pip install intel-extension-for-pytorch==2.1.100
     - name: Test with Pytest
       run: |
         pytest tests/neural_compressor/

examples/neural_compressor/language-modeling/README.md

@@ -97,4 +97,4 @@ respectively `dynamic`, `static`, `weight_only` or `aware_training`.
 
 The flag `--verify_loading` can be passed along to verify that the resulting quantized model can be loaded correctly.
 
-> **_Note:_** `weight_only` quantization_approach requires neural-compressor >= 2.3
+> **_Note:_** `weight_only` quantization_approach requires `neural-compressor` >= 2.3 and `intel-extension-for-transformers` >= 1.3.

examples/neural_compressor/language-modeling/requirements.txt

@@ -3,3 +3,5 @@ torch >= 1.9
 datasets >= 1.8.0
 sentencepiece != 0.1.92
 protobuf
+intel-extension-for-transformers >= 1.3
+peft

examples/neural_compressor/language-modeling/run_clm.py

@@ -57,6 +57,11 @@
 from transformers.utils.versions import require_version
 
 from optimum.intel.neural_compressor import INCModelForCausalLM, INCQuantizer, INCTrainer
+from optimum.intel.utils.import_utils import is_intel_extension_for_transformers_available
+
+
+if is_intel_extension_for_transformers_available():
+    from intel_extension_for_transformers.transformers.utils.config import WeightOnlyQuantConfig
 
 
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
@@ -143,7 +148,9 @@ class OptimizationArguments:
     )
     quantization_approach: str = field(
         default="dynamic",
-        metadata={"help": "Quantization approach. Supported approach are static, dynamic and aware_training."},
+        metadata={
+            "help": "Quantization approach. Supported approach are static, dynamic aware_training and weight_only."
+        },
     )
     smooth_quant: bool = field(
         default=False,
@@ -196,9 +203,13 @@ class OptimizationArguments:
         default=False,
         metadata={"help": "Whether or not to verify the loading of the quantized model."},
     )
-    bits: int = field(
-        default=8,
-        metadata={"help": "Bits for weight only quantization, 1-8 bits."},
+    bits: str = field(
+        default="4",
+        metadata={"help": "Bits number of weight for weight only quantization. 1~8 bits."},
+    )
+    weight_dtype: str = field(
+        default="int4_clip",
+        metadata={"help": "weight dtype for weight only quantization."},
     )
     group_size: int = field(
         default=-1,
@@ -214,10 +225,29 @@ class OptimizationArguments:
     )
     quantization_methodology: str = field(
         default="RTN",
+        metadata={"help": "Quantization methodology for weight only quantization. Choose from 'RTN' and 'GPTQ'."},
+    )
+    damp_percent: float = field(
+        default=0.01,
         metadata={
-            "help": "Quantization methodology for weight only quantization. Choose from 'RTN', 'AWQ' and 'GPTQ'."
+            "help": "Percentage of Hessian's diagonal values average, which will be added to Hessian's diagonal to increase numerical stability, used for GPTQ quantization"
         },
     )
+    gptq_block_size: int = field(
+        default=128,
+        metadata={"help": "Block size. sub weight matrix size to run GPTQ."},
+    )
+    num_calibration_samples: int = field(
+        default=128, metadata={"help": "Number of examples to use for the GPTQ calibration step."}
+    )
+    use_max_length: bool = field(
+        default=False,
+        metadata={"help": "Set all sequence length to be same length of args.gptq_pad_max_length"},
+    )
+    pad_max_length: int = field(
+        default=2048,
+        metadata={"help": "Calibration dataset sequence max length, this should align with your model config"},
+    )
 
 
 @dataclass
@@ -625,26 +655,33 @@ def compute_metrics(eval_preds):
             else:
                 recipes = {}
             if optim_args.quantization_approach == "weight_only":
-                op_type_dict = {
-                    ".*": {
-                        "weight": {
-                            "bits": optim_args.bits,
-                            "group_size": optim_args.group_size,
-                            "scheme": optim_args.weight_only_scheme,
-                            "algorithm": optim_args.quantization_methodology,
-                        },
-                    },
-                }
+                if not is_intel_extension_for_transformers_available():
+                    raise ImportError(
+                        "Didn't find out intel-etension-for-transformers package. "
+                        "Please install packages: pip install intel-etension-for-transformers and pip install peft."
+                    )
+                if optim_args.apply_pruning or optim_args.apply_distillation:
+                    raise ValueError("Weight only quantization and pruning or distillation cannot be combined.")
                 if optim_args.quantization_methodology == "GPTQ":
-                    gptq_args = {
-                        "pad_max_length": block_size,
+                    algorithm_args = {
+                        "act_order": False,
+                        "percdamp": optim_args.damp_percent,
+                        "block_size": optim_args.gptq_block_size,
+                        "nsamples": optim_args.num_calibration_samples,
+                        "use_max_length": optim_args.use_max_length,
+                        "pad_max_length": optim_args.pad_max_length,
                     }
-                    recipes.update({"gptq_args": gptq_args})
+                quantization_config = WeightOnlyQuantConfig(
+                    weight_dtype=optim_args.weight_dtype,
+                    group_size=optim_args.group_size,
+                    scheme=optim_args.weight_only_scheme,
+                    algorithm=optim_args.quantization_methodology,
+                    algorithm_args=algorithm_args if optim_args.quantization_methodology == "GPTQ" else None,
+                )
             else:
-                op_type_dict = {}
-            quantization_config = PostTrainingQuantConfig(
-                approach=optim_args.quantization_approach, op_type_dict=op_type_dict, recipes=recipes
-            )
+                quantization_config = PostTrainingQuantConfig(
+                    approach=optim_args.quantization_approach, recipes=recipes
+                )
 
     if optim_args.apply_pruning:
         if optim_args.end_step is None:
@@ -732,15 +769,15 @@ def compute_metrics(eval_preds):
         quantizer.quantize(
             quantization_config=quantization_config,
             save_directory=training_args.output_dir,
-            calibration_dataset=train_dataset
-            if optim_args.quantization_approach in ["static", "weight_only"]
-            else None,
-            batch_size=1  # batch_size > 1 for GPTQ is WIP
-            if optim_args.quantization_approach == "weight_only" and optim_args.quantization_methodology == "GPTQ"
-            else training_args.per_device_train_batch_size,
-            weight_only=True if optim_args.quantization_approach == "weight_only" else False,
+            calibration_dataset=(
+                train_dataset if optim_args.quantization_approach in ["static", "weight_only"] else None
+            ),
+            batch_size=(
+                1 if optim_args.quantization_approach == "weight_only" else training_args.per_device_train_batch_size
+            ),
         )
         trainer.model = quantizer._quantized_model
+
     if optim_args.apply_quantization and optim_args.verify_loading:
         loaded_model = INCModelForCausalLM.from_pretrained(training_args.output_dir)
         tokens = tokenizer("This is a sample input", return_tensors="pt")

examples/neural_compressor/text-generation/run_generation.py

@@ -368,9 +368,7 @@ def calibration_fn(p_model):
 
     args.length = adjust_length_to_model(
         args.length,
-        max_sequence_length=model.config.max_position_embeddings
-        if hasattr(model.config, "max_position_embeddings")
-        else 0,
+        max_sequence_length=getattr(model.config, "max_position_embeddings", 0),
     )
     logger.info(args)
 

optimum/intel/neural_compressor/__init__.py

@@ -12,7 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from ..utils.import_utils import is_diffusers_available
+from ..utils.import_utils import is_diffusers_available, is_intel_extension_for_transformers_available
 from .configuration import INCConfig
 from .modeling_base import (
     INCModel,

optimum/intel/neural_compressor/modeling_base.py

@@ -43,7 +43,7 @@
 from optimum.intel.generation import BaseModelForCausalLM
 
 from ...modeling_base import OptimizedModel
-from ..utils.import_utils import _torch_version, is_torch_version
+from ..utils.import_utils import _torch_version, is_intel_extension_for_transformers_available, is_torch_version
 from .configuration import INCConfig
 from .utils import WEIGHTS_NAME
 
@@ -63,6 +63,11 @@
 """
 
 
+if is_intel_extension_for_transformers_available():
+    from intel_extension_for_transformers.transformers.modeling import AutoModelForCausalLM as ITREX_WOQ_MODEL
+    from intel_extension_for_transformers.transformers.utils import WeightOnlyQuantConfig
+
+
 class INCModel(OptimizedModel):
     auto_model_class = AutoModel
     export_feature = "feature-extraction"
@@ -131,6 +136,29 @@ def _from_pretrained(
         model_save_dir = Path(model_cache_path).parent
         inc_config = None
         msg = None
+        try:
+            quantization_config = WeightOnlyQuantConfig.from_pretrained(model_id)
+            if getattr(
+                quantization_config, "algorithm", None
+            ) is not None and quantization_config.algorithm.lower() in ["rtn", "gptq", "awq", "autoaround"]:
+                if not is_intel_extension_for_transformers_available():
+                    raise ImportError(
+                        "Didn't find out intel-etension-for-transformers package. "
+                        "Please install packages: pip install intel-etension-for-transformers and pip install peft."
+                    )
+                return ITREX_WOQ_MODEL.from_pretrained(
+                    pretrained_model_name_or_path=model_id,
+                    use_auth_token=use_auth_token,
+                    revision=revision,
+                    force_download=force_download,
+                    cache_dir=cache_dir,
+                    local_files_only=local_files_only,
+                    subfolder=subfolder,
+                    trust_remote_code=trust_remote_code,
+                    **kwargs,
+                )
+        except EnvironmentError:
+            msg = "The model is not quantized with weight-only quantization."
         try:
             inc_config = INCConfig.from_pretrained(model_id)
             if not is_torch_version("==", inc_config.torch_version):