Skip to content

Commit 1032256

Browse files
AclyAcly
authored
cuda/vulkan : bicubic interpolation (#17022)
* vulkan : implement upscale with bicubic interpolation * cuda : implement upscale with bicubic interpolation * tests : add ggml_interpolate with GGML_SCALE_MODE_BICUBIC to backend tests * adapt OpenCL backend to not support the OP in that case so tests don't fail * print scale mode & flags in test-backend-ops
1 parent 15274c0 commit 1032256

File tree

5 files changed

+148
-15
lines changed

5 files changed

+148
-15
lines changed

‎ggml/src/ggml-cuda/upscale.cu‎

Lines changed: 87 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,70 @@ static __global__ void upscale_f32_bilinear(const float * x, float * dst,
8181
dst[index] = result;
8282
}
8383

84+
namespace bicubic_interpolation {
85+
// https://en.wikipedia.org/wiki/Bicubic_interpolation#Bicubic_convolution_algorithm
86+
__device__ const float a = -0.75f; // use alpha = -0.75 (same as PyTorch)
87+
88+
static __device__ float weight1(float x) { return ((a + 2) * x - (a + 3)) * x * x + 1; };
89+
static __device__ float weight2(float x) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };
90+
91+
static __device__ float bicubic(float p0, float p1, float p2, float p3, float x) {
92+
const float w0 = weight2(x + 1);
93+
const float w1 = weight1(x + 0);
94+
const float w2 = weight1(1 - x);
95+
const float w3 = weight2(2 - x);
96+
return p0 * w0 + p1 * w1 + p2 * w2 + p3 * w3;
97+
};
98+
} // namespace bicubic_interpolation
99+
100+
static __global__ void upscale_f32_bicubic(const float * x, float * dst,
101+
const int nb00, const int nb01, const int nb02, const int nb03,
102+
const int ne00_src, const int ne01_src,
103+
const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,
104+
const float sf0, const float sf1, const float sf2, const float sf3,
105+
const float pixel_offset) {
106+
using bicubic_interpolation::bicubic;
107+
108+
const int64_t index = threadIdx.x + blockIdx.x * blockDim.x;
109+
const int64_t dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;
110+
111+
if (index >= dst_total_elements) {
112+
return;
113+
}
114+
115+
const int i10_dst = index % ne10_dst;
116+
const int i11_dst = (index / ne10_dst) % ne11_dst;
117+
const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;
118+
const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);
119+
120+
const int i02_src = (int)(i12_dst / sf2);
121+
const int i03_src = (int)(i13_dst / sf3);
122+
123+
const float y_src_f = ((float)i11_dst + pixel_offset) / sf1 - pixel_offset;
124+
const int y0_src = (int)floorf(y_src_f);
125+
const float dy = y_src_f - (float)y0_src;
126+
127+
const float x_src_f = ((float)i10_dst + pixel_offset) / sf0 - pixel_offset;
128+
const int x0_src = (int)floorf(x_src_f);
129+
const float dx = x_src_f - (float)x0_src;
130+
131+
const char * x_base = (const char *)x + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03;
132+
133+
auto load = [=](int x_off, int y_off) -> float {
134+
int i00_src = max(0, min(x0_src + x_off, ne00_src - 1));
135+
int i01_src = max(0, min(y0_src + y_off, ne01_src - 1));
136+
return *(const float *)(x_base + (int64_t)i00_src * nb00 + (int64_t)i01_src * nb01);
137+
};
138+
139+
const float result = bicubic(
140+
bicubic(load(-1,-1), load(0,-1), load(1,-1), load(2,-1), dx),
141+
bicubic(load(-1, 0), load(0, 0), load(1, 0), load(2, 0), dx),
142+
bicubic(load(-1, 1), load(0, 1), load(1, 1), load(2, 1), dx),
143+
bicubic(load(-1, 2), load(0, 2), load(1, 2), load(2, 2), dx), dy);
144+
145+
dst[index] = result;
146+
}
147+
84148
static void upscale_f32_cuda(const float * x, float * dst,
85149
const int nb00, const int nb01, const int nb02, const int nb03,
86150
const int ne10, const int ne11, const int ne12, const int ne13,
@@ -104,6 +168,18 @@ static void upscale_f32_bilinear_cuda(const float * x, float * dst,
104168
upscale_f32_bilinear<<<num_blocks, CUDA_UPSCALE_BLOCK_SIZE,0,stream>>>(x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst, ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);
105169
}
106170

171+
static void upscale_f32_bicubic_cuda(const float * x, float * dst,
172+
const int nb00, const int nb01, const int nb02, const int nb03,
173+
const int ne00_src, const int ne01_src,
174+
const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,
175+
const float sf0, const float sf1, const float sf2, const float sf3,
176+
const float pixel_offset, cudaStream_t stream) {
177+
const int64_t dst_size = ne10_dst * ne11_dst * ne12_dst * ne13_dst;
178+
const int64_t num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;
179+
180+
upscale_f32_bicubic<<<num_blocks, CUDA_UPSCALE_BLOCK_SIZE,0,stream>>>(x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst, ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);
181+
}
182+
107183
void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
108184
const ggml_tensor * src0 = dst->src[0];
109185
const float * src0_d = (const float *)src0->data;
@@ -121,17 +197,22 @@ void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
121197
float sf2 = (float)dst->ne[2]/src0->ne[2];
122198
const float sf3 = (float)dst->ne[3]/src0->ne[3];
123199

200+
float pixel_offset = 0.5f;
201+
if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {
202+
sf0 = dst->ne[0] > 1 && src0->ne[0] > 1 ? (float)(dst->ne[0] - 1) / (src0->ne[0] - 1) : sf0;
203+
sf1 = dst->ne[1] > 1 && src0->ne[1] > 1 ? (float)(dst->ne[1] - 1) / (src0->ne[1] - 1) : sf1;
204+
pixel_offset = 0.0f;
205+
}
206+
124207
if (mode == GGML_SCALE_MODE_NEAREST) {
125208
upscale_f32_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, stream);
126209
} else if (mode == GGML_SCALE_MODE_BILINEAR) {
127-
float pixel_offset = 0.5f;
128-
if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {
129-
sf0 = dst->ne[0] > 1 && src0->ne[0] > 1 ? (float)(dst->ne[0] - 1) / (src0->ne[0] - 1) : sf0;
130-
sf1 = dst->ne[1] > 1 && src0->ne[1] > 1 ? (float)(dst->ne[1] - 1) / (src0->ne[1] - 1) : sf1;
131-
pixel_offset = 0.0f;
132-
}
133210
upscale_f32_bilinear_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
134211
src0->ne[0], src0->ne[1], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
135212
sf0, sf1, sf2, sf3, pixel_offset, stream);
213+
} else if (mode == GGML_SCALE_MODE_BICUBIC) {
214+
upscale_f32_bicubic_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
215+
src0->ne[0], src0->ne[1], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
216+
sf0, sf1, sf2, sf3, pixel_offset, stream);
136217
}
137218
}

‎ggml/src/ggml-opencl/ggml-opencl.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2944,8 +2944,11 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
29442944
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded
29452945
case GGML_OP_PAD:
29462946
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
2947-
case GGML_OP_UPSCALE:
2948-
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
2947+
case GGML_OP_UPSCALE: {
2948+
ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF);
2949+
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 &&
2950+
(mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR);
2951+
}
29492952
case GGML_OP_CONV_2D:
29502953
return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) ||
29512954
(op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) ||

‎ggml/src/ggml-vulkan/ggml-vulkan.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,7 @@ struct vk_device_struct {
620620
vk_pipeline pipeline_add_id_f32;
621621

622622
vk_pipeline pipeline_concat_f32, pipeline_concat_f16, pipeline_concat_i32;
623-
vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32;
623+
vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32;
624624
vk_pipeline pipeline_scale_f32;
625625
vk_pipeline pipeline_sqr_f32;
626626
vk_pipeline pipeline_sqrt_f32;
@@ -3702,6 +3702,7 @@ static void ggml_vk_load_shaders(vk_device& device) {
37023702

37033703
ggml_vk_create_pipeline(device, device->pipeline_upscale_nearest_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_NEAREST}, 1);
37043704
ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR}, 1);
3705+
ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1);
37053706

37063707
ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
37073708

@@ -8200,6 +8201,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
82008201
return ctx->device->pipeline_upscale_nearest_f32;
82018202
case GGML_SCALE_MODE_BILINEAR:
82028203
return ctx->device->pipeline_upscale_bilinear_f32;
8204+
case GGML_SCALE_MODE_BICUBIC:
8205+
return ctx->device->pipeline_upscale_bicubic_f32;
82038206
default:
82048207
return nullptr;
82058208
}

‎ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
2020
// from ggml.h: enum ggml_scale_mode, enum ggml_scale_flag
2121
#define NEAREST 0
2222
#define BILINEAR 1
23+
#define BICUBIC 2
2324

2425
layout (constant_id = 0) const uint scale_mode = 0;
2526

@@ -61,6 +62,39 @@ float interpolate_bilinear(uint i10, uint i11, uint i12, uint i13) {
6162
return fetch_bilinear(c0, c1, d, i12, i13);
6263
}
6364

65+
// Bicubic interpolation with alpha = -0.75
66+
// https://en.wikipedia.org/wiki/Bicubic_interpolation#Bicubic_convolution_algorithm
67+
const vec4 bcoeffs1 = vec4( 1.25, -2.25, 0.0, 1.0);
68+
const vec4 bcoeffs2 = vec4(-0.75, 3.75, -6.0, 3.0);
69+
vec4 powers(float x) { return vec4(x*x*x, x*x, x, 1); }
70+
71+
float bicubic(float p0, float p1, float p2, float p3, float x) {
72+
return p0 * dot(bcoeffs2, powers(x + 1)) +
73+
p1 * dot(bcoeffs1, powers(x )) +
74+
p2 * dot(bcoeffs1, powers(1 - x)) +
75+
p3 * dot(bcoeffs2, powers(2 - x));
76+
}
77+
78+
#define FETCH(a,b) data_a[base + clamp(i.x+(a), 0, res.x) * p.nb00 + clamp(i.y+(b), 0, res.y) * p.nb01]
79+
80+
float interpolate_bicubic(uint i10, uint i11, uint i12, uint i13) {
81+
const ivec2 res = ivec2(p.ne00 - 1, p.ne01 - 1);
82+
83+
const vec2 coord = (vec2(i10, i11) + p.pixel_offset) / vec2(p.sf0, p.sf1) - p.pixel_offset;
84+
const vec2 d = fract(coord);
85+
const ivec2 i = ivec2(floor(coord));
86+
87+
const uint i02 = uint(i12 / p.sf2);
88+
const uint i03 = uint(i13 / p.sf3);
89+
const uint base = p.a_offset + i03 * p.nb03 + i02 * p.nb02;
90+
91+
return bicubic(
92+
bicubic(FETCH(-1,-1), FETCH(0,-1), FETCH(1,-1), FETCH(2,-1), d.x),
93+
bicubic(FETCH(-1, 0), FETCH(0, 0), FETCH(1, 0), FETCH(2, 0), d.x),
94+
bicubic(FETCH(-1, 1), FETCH(0, 1), FETCH(1, 1), FETCH(2, 1), d.x),
95+
bicubic(FETCH(-1, 2), FETCH(0, 2), FETCH(1, 2), FETCH(2, 2), d.x), d.y);
96+
}
97+
6498
void main() {
6599
const uint idx = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
66100

@@ -81,6 +115,9 @@ void main() {
81115
case BILINEAR:
82116
result = interpolate_bilinear(i10, i11, i12, i13);
83117
break;
118+
case BICUBIC:
119+
result = interpolate_bicubic(i10, i11, i12, i13);
120+
break;
84121
}
85122

86123
data_d[p.d_offset + idx] = D_TYPE(result);

‎tests/test-backend-ops.cpp‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,10 @@ static double mean_abs_asymm(const float * a, const float * b, const size_t n, c
272272

273273
// utils for printing the variables of the test cases
274274

275+
static std::string var_to_str(const std::string & x) {
276+
return x;
277+
}
278+
275279
template<typename T>
276280
static std::string var_to_str(const T & x) {
277281
return std::to_string(x);
@@ -323,7 +327,8 @@ static std::string var_to_str(ggml_scale_mode mode) {
323327
switch (mode) {
324328
case GGML_SCALE_MODE_NEAREST: return "nearest";
325329
case GGML_SCALE_MODE_BILINEAR: return "bilinear";
326-
default: return std::to_string(mode);
330+
case GGML_SCALE_MODE_BICUBIC: return "bicubic";
331+
default: return std::to_string(mode);
327332
}
328333
}
329334

@@ -5218,7 +5223,9 @@ struct test_interpolate : public test_case {
52185223
const uint32_t mode = GGML_SCALE_MODE_NEAREST;
52195224

52205225
std::string vars() override {
5221-
return VARS_TO_STR4(type, ne, ne_tgt, mode);
5226+
ggml_scale_mode mode = (ggml_scale_mode)(this->mode & 0xFF);
5227+
std::string flags = (this->mode & GGML_SCALE_FLAG_ALIGN_CORNERS) ? "align_corners" : "none";
5228+
return VARS_TO_STR5(type, ne, ne_tgt, mode, flags);
52225229
}
52235230

52245231
test_interpolate(ggml_type type = GGML_TYPE_F32,
@@ -7288,15 +7295,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
72887295
test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2, 8, 8192, 1}, order)); // bailingmoe2 (group selection)
72897296
}
72907297

7291-
for (ggml_scale_mode mode : {GGML_SCALE_MODE_NEAREST, GGML_SCALE_MODE_BILINEAR}) {
7298+
for (ggml_scale_mode mode : {GGML_SCALE_MODE_NEAREST, GGML_SCALE_MODE_BILINEAR, GGML_SCALE_MODE_BICUBIC}) {
72927299
test_cases.emplace_back(new test_upscale(GGML_TYPE_F32, {512, 512, 3, 2}, 2, mode));
72937300
test_cases.emplace_back(new test_upscale(GGML_TYPE_F32, {512, 512, 3, 2}, 2, mode, true));
72947301
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, mode));
72957302
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {5, 7, 11, 13}, {2, 5, 7, 11}, mode));
72967303
}
7297-
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS));
7298-
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {1, 4, 3, 2}, {2, 8, 3, 2}, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS));
7299-
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {4, 1, 3, 2}, {1, 1, 3, 2}, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS));
7304+
for (ggml_scale_mode mode : {GGML_SCALE_MODE_BILINEAR, GGML_SCALE_MODE_BICUBIC}) {
7305+
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, mode | GGML_SCALE_FLAG_ALIGN_CORNERS));
7306+
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {1, 4, 3, 2}, {2, 8, 3, 2}, mode | GGML_SCALE_FLAG_ALIGN_CORNERS));
7307+
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {4, 1, 3, 2}, {1, 1, 3, 2}, mode | GGML_SCALE_FLAG_ALIGN_CORNERS));
7308+
}
73007309

73017310
test_cases.emplace_back(new test_sum());
73027311
test_cases.emplace_back(new test_sum_rows());

0 commit comments

Comments
 (0)