NeMo 2.0 を使った VLM 開発: ファインチューニングから推論、評価

torchrun --nproc_per_node=1 \
/opt/NeMo/examples/nlp/language_modeling/tuning/megatron_gpt_finetuning.py \
    exp_manager.exp_dir=${EXP_DIR} \
    exp_manager.name=${EXP_NAME} \
    trainer.precision=bf16 \
    trainer.devices=1 \
    trainer.num_nodes=1 \
    trainer.max_steps=100 \
    trainer.val_check_interval=100 \
    model.restore_from_path=${MODEL} \
    model.peft.peft_scheme="lora" \
    model.tensor_model_parallel_size=${TP_SIZE} \
    model.pipeline_model_parallel_size=${PP_SIZE}

import nemo_run as run
from nemo.collections import llm


def configure_recipe(nodes: int = 1, gpus_per_node: int = 1):
    recipe = llm.llama3_8b.finetune_recipe(
        dir="/checkpoints/llama3_finetuning",  # Path to store checkpoints
        name="llama3_lora",
        num_nodes=nodes,
        num_gpus_per_node=gpus_per_node,
        peft_scheme="lora",
    )

    recipe.trainer.max_steps = 100
    recipe.trainer.num_sanity_val_steps = 0

    # Async checkpointing doesn't work with PEFT
    recipe.trainer.strategy.ckpt_async_save = False

    # Need to set this to 1 since the default is 2
    recipe.trainer.strategy.context_parallel_size = 1
    recipe.trainer.val_check_interval = 100

    # This is currently required for LoRA/PEFT
    recipe.trainer.strategy.ddp = "megatron"

    return recipe


if __name__ == "__main__":
    run.run(configure_recipe(), executor=run.LocalExecutor())

mkdir vlm-example
cd vlm-example

sudo docker run --rm -it --gpus all --shm-size=16g --ulimit memlock=-1 --network=host -v ${PWD}:/workspace -w /workspace nvcr.io/nvidia/nemo:25.04 bash

export HF_TOKEN="hf_YOUR-HuggingFace-Token" # Change this to your Huggingface token

from nemo.collections import vlm
from nemo.collections.llm import import_ckpt

if __name__ == "__main__":
    import_ckpt(
        model=vlm.MLlamaModel(vlm.MLlamaConfig11BInstruct()),
        source="hf://meta-llama/Llama-3.2-11B-Vision-Instruct",
        output_path="/workspace/models/Llama-3.2-11B-Vision-Instruct",
    )

✓ Checkpoint imported to /workspace/models/Llama-3.2-11B-Vision-Instruct
Imported Checkpoint
├── context/
│   ├── nemo_tokenizer/
│   │   ├── special_tokens_map.json
│   │   ├── tokenizer.json
│   │   └── tokenizer_config.json
│   ├── io.json
│   └── model.yaml
└── weights/
    ├── .metadata
    ├── __0_0.distcp
    ├── __0_1.distcp
    ├── common.pt
    └── metadata.json

import os
import zipfile
import requests

from datasets import load_dataset
from tqdm import tqdm

DATA_DIR = "/workspace/dataset/"
DATASET_ID = "llm-jp/llava-instruct-ja"
URL = "http://images.cocodataset.org/zips/train2017.zip"
output_file = os.path.join(DATA_DIR, "train2017.zip")


if __name__ == "__main__":
    os.makedirs(DATA_DIR, exist_ok=True)
    dataset = load_dataset(DATASET_ID)
    dataset["train"].to_pandas().to_json(f"{DATA_DIR}/llava_instruct_ja_156k.json", orient="records", force_ascii=False, indent=2)
    dataset = dataset["train"].train_test_split(test_size=0.05, shuffle=True, seed=42)
    dataset["train"].to_json(f"{DATA_DIR}/training.jsonl", force_ascii=False)
    dataset["test"].to_json(f"{DATA_DIR}/validation.jsonl", force_ascii=False)

    response = requests.get(URL, stream=True)
    if response.status_code == 200:
	 total_size = int(response.headers.get('content-length', 0))
        block_size = 1024  # 1KB

	 with open(output_file, "wb") as file, tqdm(
            desc="Downloading train2017.zip",
            total=total_size,
            unit='B',
            unit_scale=True,
            unit_divisor=1024,
        ) as bar:
            for chunk in response.iter_content(chunk_size=block_size):
                if chunk:
                    file.write(chunk)
                    bar.update(len(chunk))
        print(f"Downloaded {output_file} successfully!")
    else:
        print("Failed to download the file.")

    extract_to = os.path.join(DATA_DIR, "coco/")
    with zipfile.ZipFile(output_file, "r") as zip_ref:
        zip_ref.extractall(extract_to)
        print(f"Extracted files to {extract_to}")

# The following is an example of a custom dataset configuration.
data_config = vlm.ImageDataConfig(
    image_folder="/workspace/dataset",
    conv_template="mllama",  # Customize based on your dataset needs
    )


class CustomMLlamaPreloadedDataModule(vlm.MLlamaPreloadedDataModule):
    def __init__(
        self,
        paths: str | List[str],
        weights: Optional[List[float]] = None,
        data_config: Optional[DataConfig] = ImageDataConfig,
        seq_length: int = 2048,
        decoder_seq_length: Optional[int] = None,
        tokenizer: Optional = None,
        image_processor: Optional = None,
        micro_batch_size: int = 4,
        global_batch_size: int = 8,
        num_train_samples: int = 10_000,
        num_val_samples: int = 10_000,
        num_test_samples: int = 10_000,
        num_workers: int = 8,
        pin_memory: bool = True,
        persistent_workers: bool = False,
        use_packed_sequence: bool = False,
        seed: int = 1234,
    ) -> None:
        super().__init__(paths=paths)
        if not isinstance(paths, (list, tuple)):
            paths = [paths]
        if weights is not None:
            assert len(weights) == len(paths)
            if len(weights) == 1:
                # weights must be None if there is only one dataset
                weights = None

        self.paths = paths
        self.weights = weights
        self.data_config = data_config
        self.seq_length = seq_length
        self.decoder_seq_length = decoder_seq_length
        self.micro_batch_size = micro_batch_size
        self.global_batch_size = global_batch_size
        self.tokenizer = tokenizer
        self.image_processor = image_processor
        self.num_train_samples = num_train_samples
        self.num_val_samples = num_val_samples
        self.num_test_samples = num_test_samples
        self.num_workers = num_workers
        self.pin_memory = pin_memory
        self.persistent_workers = persistent_workers
        self.seed = seed
        self.use_packed_sequence = use_packed_sequence
        self.init_global_step = 0
        self.tokenizer = tokenizer
        self.image_processor = image_processor

        if tokenizer is None or image_processor is None:
            logging.warning(
                "Processor and tokenizer are not provided! Fall back to `meta-llama/Llama-3.2-11B-Vision-Instruct`."
            )
            from transformers import AutoProcessor

            processor = AutoProcessor.from_pretrained(
                "meta-llama/Llama-3.2-11B-Vision-Instruct"
            )
            self.tokenizer = tokenizer or processor.tokenizer
            self.image_processor = image_processor or processor.image_processor

        self.data_sampler = MegatronDataSampler(
            seq_len=self.seq_length,
            decoder_seq_len=self.decoder_seq_length,
            micro_batch_size=micro_batch_size,
            global_batch_size=global_batch_size,
            dataloader_type="cyclic",
        )

    def setup(self, stage: str = "") -> None:
        # assert len(self.paths) == 1, "not yet support blend dataset in MLlama 2.0!"
        if self.use_packed_sequence:
            pass  # TODO
        else:
            # TODO:
            # rng = torch.Generator().manual_seed(self.seed)
            # train_dataset, val_dataset, test_dataset =
            # random_split(dataset, [train_size, val_size, test_size], generator=rng)
            self._train_ds = vlm.mllama.data.preloaded.MLlamaDataset(
                self.paths[0],
                self.data_config,
                self.tokenizer,
                self.image_processor,
                self.seq_length,
            )
            self._validation_ds = vlm.mllama.data.preloaded.MLlamaDataset(
                self.paths[1],
                self.data_config,
                self.tokenizer,
                self.image_processor,
                self.seq_length,
            )

def configure_recipe(nodes: int = 1, gpus_per_node: int = 8):
    recipe = vlm.mllama_11b.finetune_recipe(
        name="mllama_11b_finetune",
        dir="/workspace/results",
        num_nodes=nodes,
        num_gpus_per_node=gpus_per_node,
        peft_scheme="none",  # 'lora', 'none'
        )

    recipe.resume.restore_config = run.Config(
        nl.RestoreConfig,
        path="/workspace/models/Llama-3.2-11B-Vision-Instruct",
    )

    strategy = run.Config(
        nl.MegatronStrategy,
        tensor_model_parallel_size=2,
        pipeline_model_parallel_size=1,
        encoder_pipeline_model_parallel_size=0,
        pipeline_dtype=torch.bfloat16,
        )
    recipe.trainer.strategy=strategy

    recipe.trainer.max_steps = 1160
    recipe.trainer.log_every_n_steps = 1
    recipe.trainer.limit_val_batches = 1.0
    recipe.trainer.val_check_interval = 580
    recipe.log.ckpt.train_time_interval = None
    recipe.optim = distributed_fused_adam_with_cosine_annealing(
        max_lr=2.0e-5,
        min_lr=2.0e-07,
        warmup_steps=100,
        )

    recipe.data = run.Config(
        CustomMLlamaPreloadedDataModule,
        paths=[
            "/workspace/dataset/training.jsonl",
            "/workspace/dataset/validation.jsonl",
        ],  # Path to your llava-like dataset
        data_config=data_config,
        seq_length=6404,
        decoder_seq_length=2048,
        global_batch_size=128,
        micro_batch_size=1,
        tokenizer=None,  # Define your tokenizer if needed
        image_processor=None,  # Add an image processor if required
        num_workers=2,  # Number of workers for data loading
    )

    return recipe

import logging
from typing import Any, Dict, List, Optional, Sequence

import torch
import nemo_run as run
from nemo import lightning as nl
from nemo.collections import vlm
from nemo.collections.llm.recipes.optim.adam import (
    distributed_fused_adam_with_cosine_annealing,
)
from nemo.collections.vlm.neva.data.config import DataConfig, ImageDataConfig
from nemo.lightning.pytorch.plugins import MegatronDataSampler


# The following is an example of a custom dataset configuration.
data_config = vlm.ImageDataConfig(
    image_folder="/workspace/dataset",
    conv_template="mllama",  # Customize based on your dataset needs
    )


class CustomMLlamaPreloadedDataModule(vlm.MLlamaPreloadedDataModule):
    def __init__(
        self,
        paths: str | List[str],
        weights: Optional[List[float]] = None,
        data_config: Optional[DataConfig] = ImageDataConfig,
        seq_length: int = 2048,
        decoder_seq_length: Optional[int] = None,
        tokenizer: Optional = None,
        image_processor: Optional = None,
        micro_batch_size: int = 4,
        global_batch_size: int = 8,
        num_train_samples: int = 10_000,
        num_val_samples: int = 10_000,
        num_test_samples: int = 10_000,
        num_workers: int = 8,
        pin_memory: bool = True,
        persistent_workers: bool = False,
        use_packed_sequence: bool = False,
        seed: int = 1234,
    ) -> None:
        super().__init__(paths=paths)
        if not isinstance(paths, (list, tuple)):
            paths = [paths]
        if weights is not None:
            assert len(weights) == len(paths)
            if len(weights) == 1:
                # weights must be None if there is only one dataset
                weights = None

        self.paths = paths
        self.weights = weights
        self.data_config = data_config
        self.seq_length = seq_length
        self.decoder_seq_length = decoder_seq_length
        self.micro_batch_size = micro_batch_size
        self.global_batch_size = global_batch_size
        self.tokenizer = tokenizer
        self.image_processor = image_processor
        self.num_train_samples = num_train_samples
        self.num_val_samples = num_val_samples
        self.num_test_samples = num_test_samples
        self.num_workers = num_workers
        self.pin_memory = pin_memory
        self.persistent_workers = persistent_workers
        self.seed = seed
        self.use_packed_sequence = use_packed_sequence
        self.init_global_step = 0
        self.tokenizer = tokenizer
        self.image_processor = image_processor

        if tokenizer is None or image_processor is None:
            logging.warning(
                "Processor and tokenizer are not provided! Fall back to `meta-llama/Llama-3.2-11B-Vision-Instruct`."
            )
            from transformers import AutoProcessor

            processor = AutoProcessor.from_pretrained(
                "meta-llama/Llama-3.2-11B-Vision-Instruct"
            )
            self.tokenizer = tokenizer or processor.tokenizer
            self.image_processor = image_processor or processor.image_processor

        self.data_sampler = MegatronDataSampler(
            seq_len=self.seq_length,
            decoder_seq_len=self.decoder_seq_length,
            micro_batch_size=micro_batch_size,
            global_batch_size=global_batch_size,
            dataloader_type="cyclic",
        )

    def setup(self, stage: str = "") -> None:
        # assert len(self.paths) == 1, "not yet support blend dataset in MLlama 2.0!"
        if self.use_packed_sequence:
            pass  # TODO
        else:
            # TODO:
            # rng = torch.Generator().manual_seed(self.seed)
            # train_dataset, val_dataset, test_dataset =
            # random_split(dataset, [train_size, val_size, test_size], generator=rng)
            self._train_ds = vlm.mllama.data.preloaded.MLlamaDataset(
                self.paths[0],
                self.data_config,
                self.tokenizer,
                self.image_processor,
                self.seq_length,
            )
            self._validation_ds = vlm.mllama.data.preloaded.MLlamaDataset(
                self.paths[1],
                self.data_config,
                self.tokenizer,
                self.image_processor,
                self.seq_length,
            )


def configure_recipe(nodes: int = 1, gpus_per_node: int = 8):
    recipe = vlm.mllama_11b.finetune_recipe(
        name="mllama_11b_finetune",
        dir="/workspace/results",
        num_nodes=nodes,
        num_gpus_per_node=gpus_per_node,
        peft_scheme="none",  # 'lora', 'none'
        )

    recipe.resume.restore_config = run.Config(
        nl.RestoreConfig,
        path="/workspace/models/Llama-3.2-11B-Vision-Instruct",
    )

    strategy = run.Config(
        nl.MegatronStrategy,
        tensor_model_parallel_size=2,
        pipeline_model_parallel_size=1,
        encoder_pipeline_model_parallel_size=0,
        pipeline_dtype=torch.bfloat16,
        )
    recipe.trainer.strategy=strategy

    recipe.trainer.max_steps = 1160
    recipe.trainer.log_every_n_steps = 1
    recipe.trainer.limit_val_batches = 1.0
    recipe.trainer.val_check_interval = 580
    recipe.log.use_datetime_version = False
    recipe.log.ckpt.train_time_interval = None
    recipe.optim = distributed_fused_adam_with_cosine_annealing(
        max_lr=2.0e-5,
        min_lr=2.0e-07,
        warmup_steps=100,
        )

    recipe.data = run.Config(
        CustomMLlamaPreloadedDataModule,
        paths=[
            "/workspace/dataset/training.jsonl",
            "/workspace/dataset/validation.jsonl",
        ],  # Path to your llava-like dataset
        data_config=data_config,
        seq_length=6404,
        decoder_seq_length=2048,
        global_batch_size=128,
        micro_batch_size=1,
        tokenizer=None,  # Define your tokenizer if needed
        image_processor=None,  # Add an image processor if required
        num_workers=2,  # Number of workers for data loading
    )

    return recipe


if __name__ == "__main__":
    print(configure_recipe())
    run.run(configure_recipe(), direct=True)

import json
import os
import random

import webdataset as wds
from tqdm import tqdm

# Set the path to the LLaVA-Pretrain dataset directory
dataset_dir = "/workspace/dataset"

# Paths to the dataset files
json_file = os.path.join(dataset_dir, "llava_instruct_ja_156k.json")
output_path = os.path.join(dataset_dir, "wds")

if not os.path.exists(output_path):
    os.mkdir(output_path)

# Load data
with open(json_file, "r") as f:
    data = json.load(f)

random.shuffle(data)

# Convert JSON to WebDataset
with wds.ShardWriter(
    os.path.join(output_path, "instruction-%d.tar"), maxcount=10000
) as shard_writer:
    for entry in tqdm(data):
        with open(os.path.join(dataset_dir, entry["image"]), "rb") as img_file:
            image_data = img_file.read()
        sample = {
            "__key__": entry["id"],
            "jpg": image_data,
            "json": json.dumps(entry["conversations"], ensure_ascii=False).encode("utf-8"),
        }
        shard_writer.write(sample)

print("Dataset successfully converted to WebDataset format.")

> energon prepare /workspace/dataset/wds

Found 16 tar files in total. The first and last ones are:
- instruction-0.tar
- instruction-9.tar
If you want to exclude some of them, cancel with ctrl+c and specify an exclude filter in the command line.
Please enter a desired train/val/test split like "0.5, 0.2, 0.3" or "8,1,1":0.95,0.05,0 

Indexing shards  [####################################]  16/16          
Sample 0, keys:
 - jpg
 - json
Json content of sample 0 of instruction-0.tar:
[
  {
    "from": "human",
    "value": "<image>\n\u753b\u50cf\u306b\u306f\u4f55\u304c\u5199\u3063\u3066\u3044\u307e\u3059\u304b\uff1f"
  },
  {
    "from": "gpt",
    "value": "\u753b\u50cf\u306b\u306f\u3055\u307e\u3056\u307e\u306a\u7a2e\u985e\u306e\u98df\u6750\u304c\u76db\u3089\u308c\u305f\u660e\u308b\u3044\u8272\u306e\u30d7..."
  },
  {
    "from": "human",
    "value": "\u3053\u306e\u753b\u50cf\u306b\u6620\u3063\u3066\u3044\u308b\u98df\u6750\u306f\u4f55\u7a2e\u985e\u3042\u308a\u307e\u3059\u304b\uff1f"
  },
  "..."
]
Sample 1, keys:
 - jpg
 - json
Json content of sample 1 of instruction-0.tar:
[
  {
    "from": "human",
    "value": "\u3053\u306e\u753b\u50cf\u306b\u306f\u4f55\u982d\u306e\u30ad\u30ea\u30f3\u304c\u5199\u3063\u3066\u3044\u307e\u3059\u304b\uff1f\n<im..."
  },
  {
    "from": "gpt",
    "value": "\u753b\u50cf\u306b\u306f\u5c11\u306a\u304f\u3068\u30822\u982d\u306e\u30ad\u30ea\u30f3\u304c\u5199\u3063\u3066\u3044\u307e\u3059\u30021\u982d..."
  },
  {
    "from": "human",
    "value": "\u30ad\u30ea\u30f3\u306f\u4f55\u3092\u98df\u3079\u3066\u3044\u307e\u3059\u304b\uff1f"
  },
  "..."
]
Found the following part types in the dataset: json, jpg
Do you want to create a dataset.yaml interactively? [Y/n]:Y 

The following sample types are available:
0. CaptioningSample
1. ImageClassificationSample
2. ImageSample
3. InterleavedSample
4. MultiChoiceVQASample
5. OCRSample
6. Sample
7. SimilarityInterleavedSample
8. TextSample
9. VQASample
10. VidQASample
11. Crude sample (plain dict for cooking)
Please enter a number to choose a class: 9

The sample type you selected:

@dataclass
class VQASample(Sample):
    """Sample type for visual question answering."""

    #: The input image tensor in the shape (C, H, W)
    image: torch.Tensor
    #: The context/question for the image
    context: str

    #: The possible answers. Not set for testing.
    answers: Optional[List[str]] = None
    #: The weights of the possible answers. Optionally available.
    answer_weights: Optional[torch.Tensor] = None

Do you want to set a simple field_map[Y] (or write your own sample_loader [n])? [Y/n]: n

Created /workspace/dataset/wds/.nv-meta/sample_loader.py. Please edit it to return the proper values.
Done

# This file was automatically generated by `energon prepare`.
# TODO: Edit it to return the proper fields
# import torch

def sample_loader(raw: dict) -> dict:    # Note: Images are already decoded to tensors
    # TODO: Set the correct values for all (required) fields
    context = [entry['value'] for entry in raw["json"] if entry['from'] == 'human']
    answers = [entry['value'] for entry in raw["json"] if entry['from'] == 'gpt']
    return dict(
        image=raw["jpg"],  # expected type: torch.Tensor
        context=context,  # expected type: str
        answers=answers,  # expected type: typing.Optional[typing.List[str]], default: None
        answer_weights=None,  # expected type: typing.Optional[torch.Tensor], default: None
    )

def part_filter(part: str) -> bool:
    # TODO: Filter for parts required by the sample_loader
    # E.g. if your dataset contains jpeg, txt and json, but you won't use json,
    # remove it from the list, such that it is not decoded. If you need all, keep as is
    return part in ('jpg', 'json')

import logging
from typing import Any, Dict, List, Optional, Sequence

import nemo_run as run
import torch
from megatron.energon import VQASample
from nemo import lightning as nl
from nemo.collections import vlm
from nemo.collections.llm.recipes.optim.adam import (
    distributed_fused_adam_with_cosine_annealing,
)
from nemo.collections.vlm.neva.data.config import DataConfig, ImageDataConfig
from nemo.lightning.pytorch.plugins import MegatronDataSampler
from nemo.collections.multimodal.data.energon.config import MultiModalSampleConfig
from nemo.collections.vlm.mllama.data.task_encoder import LlamaTaskEncoder
from nemo.collections.multimodal.data.energon import EnergonMultiModalDataModule
from nemo.collections.multimodal.data.energon.conversation import MLlamaTemplateConfig
from nemo.collections.vlm.mllama.data.sample_encoder import Llama3SampleEncoder
from transformers import AutoProcessor
from dataclasses import dataclass, field


# Load the processor
processor = AutoProcessor.from_pretrained("meta-llama/Llama-3.2-11B-Vision-Instruct")

# Paths and configuration
data_path = "/workspace/dataset/wds"
image_processor = processor.image_processor
tokenizer = processor.tokenizer

# Define multimodal sample configuration
multimodal_sample_config = MultiModalSampleConfig(conversation_template_config=MLlamaTemplateConfig(stop_string="<|eot_id|>"))


class CustomLlama3SampleEncoder(Llama3SampleEncoder):
    """MLlama Sample Encoder"""

    def __init__(
        self,
        tokenizer,
        image_processor,
        multimodal_sample_config=MultiModalSampleConfig(),
    ):
        """
        Initialize the VQASampleEncoder.

        Parameters:
        tokenizer (Tokenizer): The HF tokenizer used for processing text.
        image_processor (ImageProcessor): The HF image processor used for preprocessing images.
        multimodal_sample_config (MultiModalSampleConfig, optional): Configuration object for multimodal samples.
            Defaults to MultiModalSampleConfig().
        """
        super().__init__(tokenizer, image_processor, multimodal_sample_config)
        self.conversation_template_config = (
            multimodal_sample_config.conversation_template_config
        )

    def process_image(self, image) -> Dict[str, torch.Tensor]:
        image_dict = self.image_processor.preprocess(image, return_tensors="pt")
        return image_dict


# Initialize the mllama task encoder
task_encoder = LlamaTaskEncoder(
    tokenizer=tokenizer,
    image_processor=image_processor,
    multimodal_sample_config=multimodal_sample_config,
)


task_encoder.encoders = {
    VQASample.__name__: CustomLlama3SampleEncoder(
        tokenizer, image_processor, multimodal_sample_config
    )
}


def configure_recipe(nodes: int = 1, gpus_per_node: int = 8):
    recipe = vlm.mllama_11b.finetune_recipe(
        name="mllama_11b_finetune_energon",
        dir="/workspace/results",
        num_nodes=nodes,
        num_gpus_per_node=gpus_per_node,
        peft_scheme="none",  # 'lora', 'none'
        )

    recipe.resume.restore_config = run.Config(
        nl.RestoreConfig,
        path="/workspace/models/Llama-3.2-11B-Vision-Instruct",
        )

    recipe.trainer.max_steps = 1143
    recipe.trainer.log_every_n_steps = 1
    recipe.trainer.limit_val_batches = 1.0
    recipe.trainer.val_check_interval = 1143
    recipe.log.use_datetime_version = False
    recipe.log.ckpt.train_time_interval = None
    recipe.optim = distributed_fused_adam_with_cosine_annealing(
        max_lr=2.0e-5,
        min_lr=2.0e-07,
        warmup_steps=100,
    )
    # Create the data module
    recipe.data = EnergonMultiModalDataModule(
        path=data_path,
        tokenizer=tokenizer,
        image_processor=image_processor,
        seq_length=6404,
        decoder_seq_length=2048,
        num_workers=8,
        micro_batch_size=1,
        global_batch_size=128,
        multimodal_sample_config=multimodal_sample_config,
        task_encoder=task_encoder,
    )

    return recipe


if __name__ == "__main__":
    print(configure_recipe())
    run.run(configure_recipe(), direct=True)

cd /opt/NeMo
curl -L https://github.com/NVIDIA/NeMo/pull/13346.diff | git apply
cd /workspace

from pathlib import Path
from nemo.collections.llm import export_ckpt

if __name__ == "__main__":
    export_ckpt(
        path=Path(
            "/workspace/results/mllama_11b_finetune/checkpoints/model_name=0--val_loss=1.00-step=1159-consumed_samples=148480.0"
            ),
        target="hf",
        output_path=Path("/workspace/results/Llama-3.2-11B-Vision-SFT"),
        overwrite=True,
    )

vllm serve "モデル path" --tensor-parallel-size {並列数} --max-num-seqs 16