根据 run_llama2_13b.yaml config运行，出现报错：

model.train(config.runner_config.epochs, dataset,
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/model.py", line 1068, in train
self._train(epoch,
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/model.py", line 114, in wrapper
func(self, *args, **kwargs)
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/model.py", line 623, in _train
self._train_dataset_sink_process(epoch, train_dataset, list_callback,
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/model.py", line 708, in _train_dataset_sink_process
outputs = train_network(*inputs)
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/cell.py", line 680, in call
out = self.compile_and_run(*args, **kwargs)
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/cell.py", line 1020, in compile_and_run
self.compile(*args, **kwargs)
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/cell.py", line 997, in compile
_cell_graph_executor.compile(self, phase=self.phase,
File "/home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/common/api.py", line 1547, in compile
result = self._graph_executor.compile(obj, args, kwargs, phase, self._use_vm_mode())
RuntimeError: Graph cycle exists, size: 1355, strike node: @4121_4113_1_mindspore_train_dataset_helper__DataWrapper_construct_144:145{[0]: ValueNode Equal, [1]: ValueNode Tensor(shape=[], dtype=Int32, value=0), [2]: @4121_4113_1_mindspore_train_dataset_helper__DataWrapper_construct_144:145{[0]: ValueNode AllReduce, [1]: 145}}

---

• Framework Unexpected Exception Raised:

---

This exception is caused by framework's unexpected error. Please create an issue at https://gitee.com/mindspore/mindspore/issues to get help.

---

• C++ Call Stack: (For framework developers)

---

mindspore/core/ir/graph_utils.cc:88 TopoSort

---

• The Traceback of Net Construct Code:

---

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/dataset_helper.py:100

def construct(self):

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/dataset_helper.py:102

    return self.network(*outputs)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/train/dataset_helper.py:102

    return self.network(*outputs)
           ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/wrapper/wrapper.py:236

    loss = self.network(*inputs)
           ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/wrapper/wrapper.py:236

    loss = self.network(*inputs)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:664

        output = self.network(*micro_input)
                 ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:664

        output = self.network(*micro_input)
                 ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama.py:461

    output = self.model(tokens, batch_valid_length, batch_index, zactivate_len, block_tables, slot_mapping)
             ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama.py:294

        h = self.layers[i](h, freqs_cis, mask, kvcache_inputs=kvcache_inputs)
            ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama.py:294

        h = self.layers[i](h, freqs_cis, mask, kvcache_inputs=kvcache_inputs)
            ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama_transformer.py:564

    h = self.attention(input_x, freqs_cis, mask, kvcache_inputs)
        ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama_transformer.py:280

    if self.qkv_concat:

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama_transformer.py:290

        query = self.cast(self.wq(x), self.dtype)  # dp, 1 -> dp, mp
                          ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/modules/layers.py:487

    if self.expert_flag:

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/modules/layers.py:487

    if self.expert_flag:

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/modules/layers.py:495

    x = self.matmul(x, weight)
        ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:517

    return self._backbone(*output)
           ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:662

    for i in range(self.micro_size):
    ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:664

        output = self.network(*micro_input)
                 ^

In file /home/ma-user/anaconda/lib/python3.9/site-packages/mindspore/nn/wrap/cell_wrapper.py:664

        output = self.network(*micro_input)
                 ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama.py:461

    output = self.model(tokens, batch_valid_length, batch_index, zactivate_len, block_tables, slot_mapping)
             ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama.py:294

        h = self.layers[i](h, freqs_cis, mask, kvcache_inputs=kvcache_inputs)
            ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama.py:294

        h = self.layers[i](h, freqs_cis, mask, kvcache_inputs=kvcache_inputs)
            ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama_transformer.py:564

    h = self.attention(input_x, freqs_cis, mask, kvcache_inputs)
        ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama_transformer.py:280

    if self.qkv_concat:

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/models/llama/llama_transformer.py:290

        query = self.cast(self.wq(x), self.dtype)  # dp, 1 -> dp, mp
                          ^

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/modules/layers.py:487

    if self.expert_flag:

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/modules/layers.py:487

    if self.expert_flag:

In file /home/ma-user/modelarts/user-job-dir/vanilla_mindformers/mindformers/modules/layers.py:495

    x = self.matmul(x, weight)
        ^

运行平台：云脑2
运行环境：版本：

mindformers 0.8.0
mindspore 2.2.0
cann:7.0

NPU: 910A