Changeset 82704 in vbox for trunk/src/VBox/Devices/Graphics/shaderlib/shader.c

Timestamp:

Jan 9, 2020 5:36:36 PM (5 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

135630

Message:

3D: Check register index in shader opcodes, bugref:9613

File:

• trunk/src/VBox/Devices/Graphics/shaderlib/shader.c (modified) (9 diffs)

trunk/src/VBox/Devices/Graphics/shaderlib/shader.c

@@ -341 +341 @@
     {
         case WINED3DSPR_TEXTURE: /* WINED3DSPR_ADDR */
-            if (shader_type == WINED3D_SHADER_TYPE_PIXEL) reg_maps->texcoord |= 1 << reg->idx;
-            else reg_maps->address |= 1 << reg->idx;
+            if (shader_type == WINED3D_SHADER_TYPE_PIXEL)
+            {
+                AssertReturn(reg->idx < MAX_REG_TEXCRD, E_INVALIDARG);
+                reg_maps->texcoord |= 1 << reg->idx;
+            }
+            else
+            {
+                AssertReturn(reg->idx < MAX_REG_ADDR, E_INVALIDARG);
+                reg_maps->address |= 1 << reg->idx;
+            }
             break;
 
         case WINED3DSPR_TEMP:
+            AssertReturn(reg->idx < MAX_REG_TEMP, E_INVALIDARG);
             reg_maps->temporary |= 1 << reg->idx;
             break;
@@ -371 +380 @@
                 }
             }
-            else reg_maps->input_registers |= 1 << reg->idx;
+            else
+            {
+                AssertReturn(reg->idx < MAX_REG_INPUT, E_INVALIDARG);
+                reg_maps->input_registers |= 1 << reg->idx;
+            }
             break;
 
@@ -404 +417 @@
             else
             {
+                AssertReturn(reg->idx < reg_maps->constf_size, E_INVALIDARG);
                 set_bitmap_bit(reg_maps->constf, reg->idx);
             }
@@ -409 +423 @@
 
         case WINED3DSPR_CONSTINT:
+            AssertReturn(reg->idx < MAX_CONST_I, E_INVALIDARG);
             reg_maps->integer_constants |= (1 << reg->idx);
             break;
 
         case WINED3DSPR_CONSTBOOL:
+            AssertReturn(reg->idx < MAX_CONST_B, E_INVALIDARG);
             reg_maps->boolean_constants |= (1 << reg->idx);
             break;
@@ -422 +438 @@
         default:
             TRACE("Not recording register of type %#x and idx %u\n", reg->type, reg->idx);
-            break;
+            return E_INVALIDARG;
     }
 
@@ -478 +494 @@
         return E_OUTOFMEMORY;
     }
+
+    reg_maps->constf_size = constf_size;
 
     while (!fe->shader_is_end(fe_data, &ptr))
@@ -667 +685 @@
 
                 hr = shader_record_register_usage(shader, reg_maps, &dst_param.reg, shader_version.type);
-                if (FAILED(hr))
-                {
-                    ERR("shader_record_register_usage failed.\n");
-                    return hr;
-                }
+                AssertReturn(FAILED(hr), hr);
 
                 /* WINED3DSPR_TEXCRDOUT is the same as WINED3DSPR_OUTPUT. _OUTPUT can be > MAX_REG_TEXCRD and
@@ -680 +694 @@
                 {
                     unsigned int idx = dst_param.reg.idx;
-                    
-                    if (idx >= MAX_REG_TEXCRD)
-                    {
-                        ERR("Invalid texcoord index %d.\n", idx);
-                        return E_INVALIDARG;
-                    }
-                    
+                    AssertReturn(idx < MAX_REG_TEXCRD, E_INVALIDARG);
+
                     reg_maps->texcoord_mask[idx] |= dst_param.write_mask;
                 }
@@ -778 +787 @@
                 struct wined3d_shader_src_param src_param, src_rel_addr;
                 unsigned int count;
+                HRESULT hr;
 
                 fe->shader_read_src_param(fe_data, &ptr, &src_param, &src_rel_addr);
                 count = get_instr_extra_regcount(ins.handler_idx, i);
 
-                shader_record_register_usage(shader, reg_maps, &src_param.reg, shader_version.type);
+                hr = shader_record_register_usage(shader, reg_maps, &src_param.reg, shader_version.type);
+                AssertReturn(FAILED(hr), hr);
+
                 while (count)
                 {
                     ++src_param.reg.idx;
-                    shader_record_register_usage(shader, reg_maps, &src_param.reg, shader_version.type);
+                    hr = shader_record_register_usage(shader, reg_maps, &src_param.reg, shader_version.type);
+                    AssertReturn(FAILED(hr), hr);
                     --count;
                 }