hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize Qwen3.8 inference on Apple silicon

Browse Source
This commit is contained in:
Georg Bauer
2026-09-04 23:27:09 +02:00
parent bf82df77cb
commit bd6353804b
14 changed files with 10149 additions and 1145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
native/metal/ds4_gpu.h
View File

@@ -43,8 +43,10 @@ typedef struct {
int ds4_gpu_init(void);
void ds4_gpu_cleanup(void);
int ds4_gpu_metal4_tensor_api_enabled(void);
ds4_gpu_tensor *ds4_gpu_tensor_alloc(uint64_t bytes);
ds4_gpu_tensor *ds4_gpu_tensor_alloc_untracked(uint64_t bytes);
ds4_gpu_tensor *ds4_gpu_tensor_alloc_managed(uint64_t bytes);
ds4_gpu_tensor *ds4_gpu_tensor_view(const ds4_gpu_tensor *base, uint64_t offset, uint64_t bytes);
void ds4_gpu_tensor_free(ds4_gpu_tensor *tensor);
@@ -130,6 +132,7 @@ int ds4_gpu_set_aux_model_map_range(const void *model_map,
int ds4_gpu_set_model_map_spans(const void *model_map, uint64_t model_size, const uint64_t *offsets, const uint64_t *sizes, uint32_t count, uint64_t max_tensor_bytes);
typedef struct {
const ds4_gpu_tensor *tensor;
const void *map;
uint64_t size;
uint64_t offset;
@@ -147,6 +150,7 @@ int ds4_gpu_qwen_dispatch(
const ds4_gpu_tensor *a,
const ds4_gpu_tensor *b,
const ds4_gpu_tensor *c,
const ds4_gpu_tensor *d,
const ds4_gpu_qwen_weight_view *weights,
uint32_t weight_count,
const ds4_gpu_qwen_kernel_args *args,
@@ -193,6 +197,7 @@ int ds4_gpu_preload_q4_expert_tables(const void *model_map, uint64_t model_size,
int ds4_gpu_should_use_managed_kv_cache(uint64_t kv_cache_bytes, uint64_t context_bytes);
void ds4_gpu_set_quality(bool quality);
void ds4_gpu_set_glm_model(bool enabled);
void ds4_gpu_set_qwen_model(bool enabled);
void ds4_gpu_set_ssd_streaming(bool enabled);
void ds4_gpu_set_glm_streaming_prefill_full_layer(bool enabled);
#ifdef __APPLE__