update documents and demo

Author: will.yang

README.md

@@ -32,55 +32,65 @@
 - [x] [TeleChat models](https://huggingface.co/Tele-AI)
 - [x] [Qwen2-VL](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct)
 - [x] [MiniCPM-V](https://huggingface.co/openbmb/MiniCPM-V-2_6)
+- [x] [DeepSeek-R1-Distill](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d)
 
 # Model Performance Benchmark
 
-| model          | dtype      | seqlen | max_context | new_tokens | TTFT(ms) | Tokens/s | memory(G) | platform |
-|:-------------- |:---------- |:------:|:-----------:|:----------:|:--------:|:--------:|:---------:|:--------:|
-| TinyLLAMA-1.1B | w4a16      | 64     | 320         | 256        | 345.00   | 21.10    | 0.77      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 410.00   | 18.50    | 0.8       | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 140.46   | 24.21    | 1.25      | RK3588   |
-|                | w8a8_g512  | 64     | 320         | 256        | 195.00   | 20.08    | 1.29      | RK3588   |
-| Qwen2-1.5B     | w4a16      | 64     | 320         | 256        | 512.00   | 14.40    | 1.75      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 550.00   | 12.75    | 1.76      | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 206.00   | 16.46    | 2.47      | RK3588   |
-|                | w8a8_g128  | 64     | 320         | 256        | 725.00   | 7.00     | 2.65      | RK3588   |
-| Phi-3-3.8B     | w4a16      | 64     | 320         | 256        | 975.00   | 6.60     | 2.16      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 1180.00  | 5.85     | 2.23      | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 516.00   | 7.44     | 3.88      | RK3588   |
-|                | w8a8_g512  | 64     | 320         | 256        | 610.00   | 6.13     | 3.95      | RK3588   |
-| ChatGLM3-6B    | w4a16      | 64     | 320         | 256        | 1168.00  | 4.62     | 3.86      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 1582.56  | 3.82     | 3.96      | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 800.00   | 4.95     | 6.69      | RK3588   |
-|                | w8a8_g128  | 64     | 320         | 256        | 2190.00  | 2.70     | 7.18      | RK3588   |
-| Gemma2-2B      | w4a16      | 64     | 320         | 256        | 628.00   | 8.00     | 3.63      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 776.20   | 7.40     | 3.63      | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 342.29   | 9.67     | 4.84      | RK3588   |
-|                | w8a8_g128  | 64     | 320         | 256        | 1055.00  | 5.49     | 5.14      | RK3588   |
-| InternLM2-1.8B | w4a16      | 64     | 320         | 256        | 475.00   | 13.30    | 1.59      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 572.00   | 11.95    | 1.62      | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 205.97   | 15.66    | 2.38      | RK3588   |
-|                | w8a8_g512  | 64     | 320         | 256        | 298.00   | 12.66    | 2.45      | RK3588   |
-| MiniCPM3-4B    | w4a16      | 64     | 320         | 256        | 1397.00  | 4.80     | 2.7       | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 1645.00  | 4.39     | 2.8       | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 702.18   | 6.15     | 4.65      | RK3588   |
-|                | w8a8_g128  | 64     | 320         | 256        | 1691.00  | 3.42     | 5.06      | RK3588   |
-| llama3-8B      | w4a16      | 64     | 320         | 256        | 1607.98  | 3.60     | 5.63      | RK3576   |
-|                | w4a16_g128 | 64     | 320         | 256        | 2010.00  | 3.00     | 5.76      | RK3576   |
-|                | w8a8       | 64     | 320         | 256        | 1128.00  | 3.79     | 9.21      | RK3588   |
-|                | w8a8_g512  | 64     | 320         | 256        | 1281.35  | 3.05     | 9.45      | RK3588   |
+| llm model      | dtype      | seqlen | max_context | new_tokens | TTFT(ms) | Tokens/s | memory(G) | platform |
+| :------------- | :--------- | :----: | :---------: | :--------: | :------: | :------: | :-------: | :------: |
+| TinyLLAMA-1.1B | w4a16      |   64   |     320     |    256     |  345.00  |  21.10   |   0.77    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     |  410.00  |  18.50   |    0.8    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  140.46  |  24.21   |   1.25    |  RK3588  |
+|                | w8a8_g512  |   64   |     320     |    256     |  195.00  |  20.08   |   1.29    |  RK3588  |
+| Qwen2-1.5B     | w4a16      |   64   |     320     |    256     |  512.00  |  14.40   |   1.75    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     |  550.00  |  12.75   |   1.76    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  206.00  |  16.46   |   2.47    |  RK3588  |
+|                | w8a8_g128  |   64   |     320     |    256     |  725.00  |   7.00   |   2.65    |  RK3588  |
+| Phi-3-3.8B     | w4a16      |   64   |     320     |    256     |  975.00  |   6.60   |   2.16    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     | 1180.00  |   5.85   |   2.23    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  516.00  |   7.44   |   3.88    |  RK3588  |
+|                | w8a8_g512  |   64   |     320     |    256     |  610.00  |   6.13   |   3.95    |  RK3588  |
+| ChatGLM3-6B    | w4a16      |   64   |     320     |    256     | 1168.00  |   4.62   |   3.86    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     | 1582.56  |   3.82   |   3.96    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  800.00  |   4.95   |   6.69    |  RK3588  |
+|                | w8a8_g128  |   64   |     320     |    256     | 2190.00  |   2.70   |   7.18    |  RK3588  |
+| Gemma2-2B      | w4a16      |   64   |     320     |    256     |  628.00  |   8.00   |   3.63    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     |  776.20  |   7.40   |   3.63    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  342.29  |   9.67   |   4.84    |  RK3588  |
+|                | w8a8_g128  |   64   |     320     |    256     | 1055.00  |   5.49   |   5.14    |  RK3588  |
+| InternLM2-1.8B | w4a16      |   64   |     320     |    256     |  475.00  |  13.30   |   1.59    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     |  572.00  |  11.95   |   1.62    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  205.97  |  15.66   |   2.38    |  RK3588  |
+|                | w8a8_g512  |   64   |     320     |    256     |  298.00  |  12.66   |   2.45    |  RK3588  |
+| MiniCPM3-4B    | w4a16      |   64   |     320     |    256     | 1397.00  |   4.80   |    2.7    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     | 1645.00  |   4.39   |    2.8    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     |  702.18  |   6.15   |   4.65    |  RK3588  |
+|                | w8a8_g128  |   64   |     320     |    256     | 1691.00  |   3.42   |   5.06    |  RK3588  |
+| llama3-8B      | w4a16      |   64   |     320     |    256     | 1607.98  |   3.60   |   5.63    |  RK3576  |
+|                | w4a16_g128 |   64   |     320     |    256     | 2010.00  |   3.00   |   5.76    |  RK3576  |
+|                | w8a8       |   64   |     320     |    256     | 1128.00  |   3.79   |   9.21    |  RK3588  |
+|                | w8a8_g512  |   64   |     320     |    256     | 1281.35  |   3.05   |   9.45    |  RK3588  |
+
+| multimodal model | image input size | vision model dtype | vision infer time(s) | vision memory(MB) | llm model dtype | seqlen | max_context | new_tokens | TTFT(ms) | Tokens/s | llm memory(G) | platform |
+|:-------------- |:---------- |:------:|:-----------:|:----------:|:--------:|:--------:|:---------:|:--------:|:---------:|:---------:|:---------:|:---------:|
+| Qwen2-VL-2B | (1, 3, 392, 392) | fp16 | 3.55 | 1436.52 | w4a16 | 256 | 384 | 128 | 2094.17 | 13.23 | 1.75 | RK3576 |
+|                              |    | fp16  | 3.28 | 1436.52 | w8a8 | 256 | 384 | 128 | 856.86 | 16.19 | 2.47 | RK3588 |
+| MiniCPM-V-2_6 | (1, 3, 448, 448) | fp16 | 2.40 | 1031.30 | w4a16 | 128 | 256 | 128 | 2997.70 | 3.84 | 5.50 | RK3576 |
+|                            |    | fp16  | 3.27 | 976.98 | w8a8 | 128 | 256 | 128 | 1720.60 | 4.13 | 8.88 | RK3588 |
 
 - This performance data were collected based on the maximum CPU and NPU frequencies of each platform with version 1.1.0. 
 - The script for setting the frequencies is located in the scripts directory.
+- The vision model were tested based on all NPU core with rknn-toolkit2 version 2.2.0.
 
 # Download
 
-You can download the latest package, docker image, example, documentation, and platform-tool from [RKLLM_SDK](https://console.zbox.filez.com/l/RJJDmB), fetch code: rkllm
+1. You can download the **latest package** from [RKLLM_SDK](https://console.zbox.filez.com/l/RJJDmB), fetch code: rkllm
+2. You can download the **converted rkllm model**  from [rkllm_model_zoo](https://console.box.lenovo.com/l/l0tXb8), fetch code: rkllm
 
 # Examples
 
-1. Multimodel deployment demo:   [rkllm_multimodel_demo](https://github.com/airockchip/rknn-llm/tree/main/examples/rkllm_multimodel_demo)
-2. API usage demo:  [rkllm_api_demo](https://github.com/airockchip/rknn-llm/tree/main/examples/rkllm_api_demo)
+1. Multimodel deployment demo:   [Qwen2-VL-2B_Demo](https://github.com/airockchip/rknn-llm/tree/main/examples/Qwen2-VL-2B_Demo)
+2. API usage demo:  [DeepSeek-R1-Distill-Qwen-1.5B_Demo](https://github.com/airockchip/rknn-llm/tree/main/examples/DeepSeek-R1-Distill-Qwen-1.5B_Demo)
 3. API server demo:  [rkllm_server_demo](https://github.com/airockchip/rknn-llm/tree/main/examples/rkllm_server_demo)
 
 # Note
@@ -117,4 +127,4 @@ https://github.com/airockchip/rknn-toolkit2
 - Add support for models such as Llama3, Gemma2, and MiniCPM3.
 - Resolve catastrophic forgetting issue when the number of tokens exceeds max_context.
 
-for older version, please refer [CHANGELOG](CHANGELOG.md)
+for older version, please refer [CHANGELOG](CHANGELOG.md)

doc/Rockchip_RKLLM_SDK_CN_1.1.0.pdf

@@ -0,0 +1,194 @@
+# DeepSeek-R1-Distill-Qwen-1.5B Demo
+1. This demo demonstrates how to deploy the DeepSeek-R1-Distill-Qwen-1.5B model.
+2. The open-source model used in this demo is available at: [DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B)
+
+## 1. Requirements
+
+```
+rkllm-toolkit==1.1.4
+rkllm-runtime==1.1.4
+python==3.8 or python==3.10
+```
+
+## 2. Model Conversion
+
+1. Firstly, you need to create `data_quant.json` for quantizing the rkllm model, we use fp16 model generation results as the quantization calibration data.
+2. Secondly, you run the following code to generate `data_quant.json`  and export the rkllm model.
+3. You can also download the **converted rkllm model**  from [rkllm_model_zoo](https://console.box.lenovo.com/l/l0tXb8), fetch code: rkllm
+
+```bash
+cd export
+python generate_data_quant.py -m /path/to/DeepSeek-R1-Distill-Qwen-1.5B
+python export_rkllm.py
+```
+
+## 3. C++ Demo
+
+In the `deploy` directory, we provide example code for board-side inference. 
+
+### 1. Compile and Build
+
+Users can directly compile the example code by running the `deploy/build-linux.sh` or `deploy/build-android.sh` script (replacing the cross-compiler path with the actual path). This will generate an `install/demo_Linux_aarch64` folder in the `deploy` directory, containing the executables `llm_demo`, and the `lib` folder.
+
+```bash
+cd deploy
+# for linux
+./build-linux.sh
+# for android
+./build-android.sh
+# push install dir to device
+adb push install/demo_Linux_aarch64 /data
+# push model file to device
+adb push DeepSeek-R1-Distill-Qwen-1.5B.rkllm /data/demo_Linux_aarch64
+```
+
+### 2. Run Demo
+
+Enter the `/data/demo_Linux_aarch64` directory on the board and run the example using the following code
+
+```bash
+adb shell
+cd /data/demo_Linux_aarch64
+# export lib path
+export LD_LIBRARY_PATH=./lib
+taskset f0 ./llm_demo /path/to/your/rkllm/model 2048 4096
+
+# Running result                                                          
+rkllm init start
+rkllm init success
+
+**********************可输入以下问题对应序号获取回答/或自定义输入********************
+
+[0] 现有一笼子，里面有鸡和兔子若干只，数一数，共有头14个，腿38条，求鸡和兔子各有多少只？
+[1] 有28位小朋友排成一行,从左边开始数第10位是学豆,从右边开始数他是第几位?
+
+*************************************************************************
+
+
+user:
+```
+
+example 1 （DeepSeek-R1-Distill-Qwen-1.5B_W8A8_RK3588.rkllm）
+
+```
+user: 0
+现有一笼子，里面有鸡和兔子若干只，数一数，共有头14个，腿38条，求鸡和兔子各有多少只？
+robot: <think>
+首先，设鸡的数量为x，兔子的数量为y。
+
+根据题目中的条件，我们知道：
+
+1. 鸡和兔子的总数是14，因此有方程：
+   x + y = 14
+
+2. 鸡有两条腿，兔子有四条腿，总腿数是38，所以有另一个方程：
+   2x + 4y = 38
+
+接下来，通过代入法或消元法来解这两个方程。假设我们用代入法：
+
+从第一个方程中，可以得到：
+x = 14 - y
+
+将这个表达式代入第二个方程：
+2(14 - y) + 4y = 38
+展开计算后得到：
+28 - 2y + 4y = 38
+合并同类项：
+2y = 10
+解得：
+y = 5
+
+然后，将y的值代入x = 14 - y中：
+x = 14 - 5 = 9
+
+因此，鸡有9只，兔子有5只。
+</think>
+
+要解决这个问题，我们可以设鸡的数量为 \( x \)，兔子的数量为 \( y \)。根据题目给出的条件：
+
+1. **头的总数**：每只鸡和兔子都有一个头，所以：
+   \[
+   x + y = 14
+   \]
+
+2. **腿的总数**：鸡有两条腿，兔子有四条腿，总腿数为38条，因此：
+   \[
+   2x + 4y = 38
+   \]
+
+接下来，我们可以通过解这两个方程来找到 \( x \) 和 \( y \) 的值。
+
+**步骤一：简化第二个方程**
+
+将第二个方程两边同时除以2：
+\[
+x + 2y = 19
+\]
+
+现在，我们有两个方程：
+\[
+\begin{cases}
+x + y = 14 \\
+x + 2y = 19
+\end{cases}
+\]
+
+**步骤二：消元法**
+
+用第二个方程减去第一个方程：
+\[
+(x + 2y) - (x + y) = 19 - 14 \\
+y = 5
+\]
+
+**步骤三：代入求 \( x \)**
+
+将 \( y = 5 \) 代入第一个方程：
+\[
+x + 5 = 14 \\
+x = 14 - 5 \\
+x = 9
+\]
+
+因此，鸡的数量是 **9只**，兔子的数量是 **5只**。
+
+**最终答案：**
+鸡有 \(\boxed{9}\) 只，兔子有 \(\boxed{5}\) 只。
+```
+
+example 2 （DeepSeek-R1-Distill-Qwen-1.5B_W8A8_RK3588.rkllm）
+
+```
+
+user: 1
+有28位小朋友排成一行,从左边开始数第10位是学豆,从右边开始数他是第几位?
+robot: <think>
+首先，总共有28位小朋友。
+
+从左边开始数，第10位是学豆的位置。
+
+因此，从右边开始数，学豆的位置是从右边数的第(28 - 10 + 1) = 第19位。
+</think>
+
+**解答：**
+
+我们有28位小朋友排成一行。题目要求确定从右边开始数时，第10位是学豆的位置。
+
+**步骤如下：**
+
+1. **总人数**：共有28位小朋友。
+2. **左边数的顺序**：从左边开始数，第10位是学豆。
+3. **右边数的计算**：
+   - 从右边数时，第1位对应左边数的第28位。
+   - 因此，第n位在左边对应的是第(28 - n + 1)位在右边。
+
+4. **具体计算**：
+   \[
+   第10位在左边 = 第(28 - 10 + 1) = 第19位在右边
+   \]
+
+**最终答案：**
+
+\boxed{19}
+```
+

doc/Rockchip_RKLLM_SDK_CN_1.1.4.pdf

@@ -0,0 +1,37 @@
+cmake_minimum_required(VERSION 3.10)
+project(rkllm_demo)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Android")
+    set (TARGET_LIB_ARCH ${CMAKE_ANDROID_ARCH_ABI})
+else()
+    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+        set (TARGET_LIB_ARCH aarch64)
+    else()
+        set (TARGET_LIB_ARCH armhf)
+    endif()
+    if (CMAKE_C_COMPILER MATCHES "uclibc")
+        set (TARGET_LIB_ARCH ${TARGET_LIB_ARCH}_uclibc)
+    endif()
+endif()
+
+set(SOURCE_FILES_1 src/llm_demo.cpp)
+add_executable(llm_demo ${SOURCE_FILES_1})
+
+set(RKLLM_API_PATH "${CMAKE_SOURCE_DIR}/../../../rkllm-runtime/${CMAKE_SYSTEM_NAME}/librkllm_api")
+include_directories(${RKLLM_API_PATH}/include)
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+    set(RKLLM_RT_LIB ${RKLLM_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librkllmrt.so)
+    find_package(OpenMP REQUIRED)
+    target_link_libraries(llm_demo  ${RKLLM_RT_LIB} log OpenMP::OpenMP_CXX)
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    set(RKLLM_RT_LIB ${RKLLM_API_PATH}/aarch64/librkllmrt.so)
+    target_link_libraries(llm_demo  ${RKLLM_RT_LIB})
+endif()
+
+# Install the executable file to the specified directory
+set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install/demo_${CMAKE_SYSTEM_NAME}_${TARGET_LIB_ARCH})
+install(TARGETS llm_demo DESTINATION ./)
+install(PROGRAMS ${RKLLM_RT_LIB} DESTINATION lib)

doc/Rockchip_RKLLM_SDK_EN_1.1.0.pdf doc/Rockchip_RKLLM_SDK_EN_1.1.4.pdfdoc/Rockchip_RKLLM_SDK_EN_1.1.0.pdf renamed to doc/Rockchip_RKLLM_SDK_EN_1.1.4.pdf

@@ -4,7 +4,7 @@ if [[ -z ${BUILD_TYPE} ]];then
     BUILD_TYPE=Release
 fi
 
-ANDROID_NDK_PATH=~/opts/ndk/android-ndk-r21e
+ANDROID_NDK_PATH=~/opts/android-ndk-r21e
 TARGET_ARCH=arm64-v8a
 
 TARGET_PLATFORM=android
@@ -30,4 +30,5 @@ cmake ../.. \
     -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
     -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
 
-make -j4
+make -j4
+make install

examples/DeepSeek-R1-Distill-Qwen-1.5B_Demo/Readme.md

@@ -31,4 +31,5 @@ cmake ../.. \
     -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
     -DCMAKE_POSITION_INDEPENDENT_CODE=ON
 
-make -j4
+make -j4
+make install