diff --git a/packages/compiler-sfc/__tests__/compileScriptRefSugar.spec.ts b/packages/compiler-sfc/__tests__/compileScriptRefSugar.spec.ts
index 59964d85..53172fa7 100644
--- a/packages/compiler-sfc/__tests__/compileScriptRefSugar.spec.ts
+++ b/packages/compiler-sfc/__tests__/compileScriptRefSugar.spec.ts
@@ -366,5 +366,26 @@ describe('<script setup> ref sugar', () => {
         )
       ).toThrow(`cannot reference locally declared variables`)
     })
+
+    test('warn usage in non-init positions', () => {
+      expect(() =>
+        compile(
+          `<script setup>
+      let bar = $ref(1)
+      bar = $ref(2)
+    </script>`,
+          { refSugar: true }
+        )
+      ).toThrow(`$ref can only be used directly as a variable initializer`)
+
+      expect(() =>
+        compile(
+          `<script setup>
+      let bar = { foo: $computed(1) }
+    </script>`,
+          { refSugar: true }
+        )
+      ).toThrow(`$computed can only be used directly as a variable initializer`)
+    })
   })
 })
diff --git a/packages/compiler-sfc/src/compileScript.ts b/packages/compiler-sfc/src/compileScript.ts
index 9a558f25..000dff41 100644
--- a/packages/compiler-sfc/src/compileScript.ts
+++ b/packages/compiler-sfc/src/compileScript.ts
@@ -1096,7 +1096,7 @@ export function compileScript(
 
   // 3. Do a full walk to rewrite identifiers referencing let exports with ref
   // value access
-  if (enableRefSugar && Object.keys(refBindings).length) {
+  if (enableRefSugar) {
     const onIdent = (id: Identifier, parent: Node, parentStack: Node[]) => {
       if (refBindings[id.name] && !refIdentifiers.has(id)) {
         if (isStaticProperty(parent) && parent.shorthand) {
@@ -1115,13 +1115,26 @@ export function compileScript(
       }
     }
 
-    const onNode = (node: Node) => {
+    const onNode = (node: Node, parent: Node) => {
       if (isCallOf(node, $RAW)) {
         s.remove(
           node.callee.start! + startOffset,
           node.callee.end! + startOffset
         )
         return false // skip walk
+      } else if (
+        parent &&
+        isCallOf(
+          node,
+          id => id === $REF || id === $FROM_REFS || id === $COMPUTED
+        ) &&
+        (parent.type !== 'VariableDeclarator' || node !== parent.init)
+      ) {
+        error(
+          // @ts-ignore
+          `${node.callee.name} can only be used directly as a variable initializer.`,
+          node
+        )
       }
     }