refactor(tokens): Update JSON files to support Tokens Studio management

packages/calcite-design-tokens/src/build-tokens.ts

@@ -1,3 +1,5 @@
-import { semantic } from "./build/dictionaries/index.js";
+import { dark, light, semantic } from "./build/dictionaries/index.js";
 
+await dark.buildAllPlatforms();
+await light.buildAllPlatforms();
 await semantic.buildAllPlatforms();

packages/calcite-design-tokens/src/build/dictionaries/index.ts

@@ -1,7 +1,7 @@
 import StyleDictionary from "style-dictionary";
 import { register as registerTokenStudioTransformers } from "@tokens-studio/sd-transforms";
-import calciteLightConfig from "../../config/calcite/light.js";
-import calciteDarkConfig from "../../config/calcite/dark.js";
+import calciteLightConfig from "../../config/color/light.js";
+import calciteDarkConfig from "../../config/color/dark.js";
 import semanticConfig from "../../config/semantic.js";
 import { register as registerCalciteHooks } from "../registry/index.js";
 import { applyBuiltInOverrides, applyOverrides } from "../overrides/index.js";
@@ -14,9 +14,9 @@ await registerTokenStudioTransformers(StyleDictionary, {
 await registerCalciteHooks(StyleDictionary);
 applyOverrides(StyleDictionary);
 
-const semantic = new StyleDictionary(semanticConfig);
 const light = new StyleDictionary(calciteLightConfig);
 const dark = new StyleDictionary(calciteDarkConfig);
+const semantic = new StyleDictionary(semanticConfig);
 
 applyBuiltInOverrides([semantic, light, dark]);
 

packages/calcite-design-tokens/src/build/filter/core.ts

@@ -1,8 +1,10 @@
 import { Filter } from "style-dictionary/types";
 import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
+import { isLightOrDarkColorToken } from "./light-or-dark.js";
 
-export const filterCoreTokens: Filter["filter"] = (token) => !token.isSource && token.type !== "typography";
+export const filterCoreTokens: Filter["filter"] = (token, config) =>
+  !token.isSource && token.type !== "typography" && !isLightOrDarkColorToken(token, config);
 
 export const registerFilterCoreTokens: RegisterFn = () =>
   StyleDictionary.registerFilter({

packages/calcite-design-tokens/src/build/filter/dark.ts

@@ -3,10 +3,7 @@ import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
 import { isThemed } from "../utils/token-types.js";
 
-export const filterDarkColorTokens: Filter["filter"] = (token) =>
-  token.isSource &&
-  (isThemed(token, { theme: "dark", targetPropName: "filePath" }) ||
-    isThemed(token, { theme: "dark", targetPropName: "path" }));
+export const filterDarkColorTokens: Filter["filter"] = (token) => isThemed(token, { theme: "dark" });
 
 export const registerFilterDarkColorTokens: RegisterFn = () =>
   StyleDictionary.registerFilter({

packages/calcite-design-tokens/src/build/filter/global-js.ts

@@ -1,14 +1,16 @@
 import { Filter } from "style-dictionary/types";
 import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
+import { isLightOrDarkColorToken } from "./light-or-dark.js";
 
 /**
  * This filter helps match the test snapshot, this can be removed once outputs are consolidated
  *
  * @param token
+ * @param config
  */
-export const filterGlobalTokensJs: Filter["filter"] = (token) => {
-  return token.isSource;
+export const filterGlobalTokensJs: Filter["filter"] = (token, config) => {
+  return token.isSource || isLightOrDarkColorToken(token, config);
 };
 
 export const registerFilterGlobalTokensJs: RegisterFn = () =>

packages/calcite-design-tokens/src/build/filter/global.ts

@@ -2,11 +2,13 @@ import { Filter } from "style-dictionary/types";
 import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
 import { isBreakpoint } from "../utils/token-types.js";
+import { state } from "../shared/state.js";
 import { isLightOrDarkColorToken } from "./light-or-dark.js";
 
 export const filterGlobalTokens: Filter["filter"] = (token, config) => {
   return (
-    token.isSource && !(token.type === "typography" || isBreakpoint(token) || isLightOrDarkColorToken(token, config))
+    state.sameValueThemeTokens.has(token.key) ||
+    (token.isSource && !(token.type === "typography" || isBreakpoint(token) || isLightOrDarkColorToken(token, config)))
   );
 };
 

packages/calcite-design-tokens/src/build/filter/include.ts

@@ -1,8 +1,9 @@
 import { Filter } from "style-dictionary/types";
 import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
+import { isThemed } from "../utils/token-types.js";
 
-export const filterIncludeTokens: Filter["filter"] = (token) => !token.isSource;
+export const filterIncludeTokens: Filter["filter"] = (token) => !token.isSource && !isThemed(token);
 
 export const registerFilterIncludeTokens: RegisterFn = () => {
   StyleDictionary.registerFilter({

packages/calcite-design-tokens/src/build/filter/light-or-dark.ts

@@ -3,11 +3,9 @@ import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
 import { isThemed } from "../utils/token-types.js";
 
-export const isLightOrDarkColorToken: Filter["filter"] = (token) =>
-  isThemed(token, { targetPropName: "filePath" }) || isThemed(token, { targetPropName: "path" });
+export const isLightOrDarkColorToken: Filter["filter"] = (token) => isThemed(token);
 
-const filterLightOrDarkColorTokens: Filter["filter"] = (token) =>
-  token.isSource && (isThemed(token, { targetPropName: "filePath" }) || isThemed(token, { targetPropName: "path" }));
+const filterLightOrDarkColorTokens: Filter["filter"] = (token) => !token.isSource && isThemed(token);
 
 export const registerFilterLightOrDarkColorTokens: RegisterFn = () =>
   StyleDictionary.registerFilter({

packages/calcite-design-tokens/src/build/filter/light.ts

@@ -3,10 +3,7 @@ import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../types/interfaces.js";
 import { isThemed } from "../utils/token-types.js";
 
-export const filterLightColorTokens: Filter["filter"] = (token) =>
-  token.isSource &&
-  (isThemed(token, { theme: "light", targetPropName: "filePath" }) ||
-    isThemed(token, { theme: "light", targetPropName: "path" }));
+export const filterLightColorTokens: Filter["filter"] = (token) => isThemed(token, { theme: "light" });
 
 export const registerFilterLightColorTokens: RegisterFn = () =>
   StyleDictionary.registerFilter({

packages/calcite-design-tokens/src/build/format/index-file.ts

@@ -1,9 +1,11 @@
 import prettierSync from "@prettier/sync";
-import { Dictionary, FormatFn, FormatFnArguments, TransformedTokens } from "style-dictionary/types";
+import { Dictionary, FormatFn, FormatFnArguments } from "style-dictionary/types";
 import { fileHeader, formattedVariables } from "style-dictionary/utils";
 import StyleDictionary from "style-dictionary";
 import { RegisterFn, Stylesheet } from "../types/interfaces.js";
 import { fromTokens } from "../utils/dictionary.js";
+import { isThemed } from "../utils/token-types.js";
+import { dark, light } from "../dictionaries/index.js";
 
 export const registerFormatIndex: RegisterFn = () => {
   StyleDictionary.registerFormat({
@@ -13,7 +15,7 @@ export const registerFormatIndex: RegisterFn = () => {
 };
 
 export const formatIndexFile: FormatFn = async (args) => {
-  const { dictionary, file, options } = args;
+  const { file, options } = args;
 
   if (options.fileExtension !== ".css" && options.fileExtension !== ".scss") {
     throw new Error("Only .css and .scss file extensions are supported.");
@@ -23,22 +25,21 @@ export const formatIndexFile: FormatFn = async (args) => {
   const themes = ["light", "dark"] as const;
   const format = options.fileExtension.replace(".", "") as Stylesheet;
 
-  const themedTokens = dictionary.allTokens.reduce(
-    (acc: { light: TransformedTokens; dark: TransformedTokens }, token) => {
-      if (token.path.includes("light")) {
-        acc.light[token.name] = token;
-      } else if (token.path.includes("dark")) {
-        acc.dark[token.name] = token;
-      }
-      return acc;
-    },
-    { light: {}, dark: {} },
-  );
+  const darkDictionary = await dark.getPlatformTokens(options.platform, { cache: true });
+  const lightDictionary = await light.getPlatformTokens(options.platform, { cache: true });
 
   const commonVarFormat = "css";
   const varLists = {
-    light: createVarList(commonVarFormat, fromTokens(themedTokens.light), args),
-    dark: createVarList(commonVarFormat, fromTokens(themedTokens.dark), args),
+    light: createVarList(
+      commonVarFormat,
+      fromTokens(lightDictionary.allTokens.filter((token) => isThemed(token, { theme: "light" }))),
+      args,
+    ),
+    dark: createVarList(
+      commonVarFormat,
+      fromTokens(darkDictionary.allTokens.filter((token) => isThemed(token, { theme: "dark" }))),
+      args,
+    ),
   } as const;
 
   const classGroupStrategy = format === "scss" ? "@mixin " : ".";

packages/calcite-design-tokens/src/build/preprocessors/index.ts

@@ -1,7 +1,10 @@
+import { registerPreprocessorStoreSameValueThemeTokens } from "./store-same-value-theme-tokens.js";
 import { registerPreprocessorStorePostMergeDictionary } from "./store-post-merge-dictionary.js";
 
 export function registerPreprocessors(): void {
   registerPreprocessorStorePostMergeDictionary();
+  registerPreprocessorStoreSameValueThemeTokens();
 }
 
+export * from "./store-same-value-theme-tokens.js";
 export * from "./store-post-merge-dictionary.js";

packages/calcite-design-tokens/src/build/preprocessors/store-same-value-theme-tokens.ts

@@ -0,0 +1,29 @@
+import StyleDictionary from "style-dictionary";
+import { DesignToken } from "style-dictionary/types";
+import { dark, light } from "../dictionaries/index.js";
+import { isThemed } from "../utils/token-types.js";
+import { state } from "../shared/state.js";
+
+export function registerPreprocessorStoreSameValueThemeTokens(): void {
+  StyleDictionary.registerPreprocessor({
+    name: PreprocessorStoreSameValueThemeTokens,
+    preprocessor: async (dictionary) => {
+      const sameValueThemeTokens = light.allTokens
+        .filter((token) => isThemed(token))
+        .filter((token, index) => token.value === dark.allTokens[index].value);
+      const keyToToken = new Map<string, DesignToken>();
+
+      sameValueThemeTokens.forEach((token) => {
+        if (token.key) {
+          keyToToken.set(token.key, token);
+        }
+      });
+
+      state.sameValueThemeTokens = keyToToken;
+
+      return dictionary;
+    },
+  });
+}
+
+export const PreprocessorStoreSameValueThemeTokens = "calcite/preprocessor/store-same-value-theme-tokens";

packages/calcite-design-tokens/src/build/shared/state.ts

@@ -1,5 +1,12 @@
-const sharedBuildState: any = {
-  postMergeDictionary: undefined,
-} as const;
+import { DesignToken, PreprocessedTokens } from "style-dictionary/types";
+
+type State = {
+  postMergeDictionary: PreprocessedTokens;
+  sameValueThemeTokens: Map<string, DesignToken>;
+};
+
+const sharedBuildState = {
+  // populated by preprocessors
+} as unknown as State;
 
 export { sharedBuildState as state };

packages/calcite-design-tokens/src/build/transforms/value/ensure-type.ts

@@ -2,10 +2,11 @@ import { Filter, ValueTransform } from "style-dictionary/types";
 import { ArrayValues } from "type-fest";
 import StyleDictionary from "style-dictionary";
 import { RegisterFn } from "../../types/interfaces.js";
+import { isBreakpoint } from "../../utils/token-types.js";
 
 const stringValueTokenTypes = ["fontWeight", "dimension", "lineHeight", "opacity", "z-index"] as const;
 const filterTypes: Filter["filter"] = (token) =>
-  stringValueTokenTypes.includes(token.type as ArrayValues<typeof stringValueTokenTypes>);
+  stringValueTokenTypes.includes(token.type as ArrayValues<typeof stringValueTokenTypes>) && !isBreakpoint(token);
 
 const transformValueEnsureType: ValueTransform["transform"] = async (token) => {
   return `${token.value}`;

packages/calcite-design-tokens/src/build/transforms/value/merge-value.ts

@@ -29,10 +29,15 @@ const transformValueMergeValues: ValueTransform["transform"] = async (token, con
   );
 
   if (tokenIndex > -1) {
-    return {
-      dark: dictionaries.dark.allTokens[tokenIndex].value,
-      light: dictionaries.light.allTokens[tokenIndex].value,
-    };
+    const lightValue = dictionaries.light.allTokens[tokenIndex].value;
+    const darkValue = dictionaries.dark.allTokens[tokenIndex].value;
+
+    if (lightValue !== darkValue) {
+      return {
+        light: lightValue,
+        dark: darkValue,
+      };
+    }
   }
 
   return token.value;

packages/calcite-design-tokens/src/build/transforms/value/px-to-rem.ts

@@ -32,15 +32,15 @@ function throwSizeError(name: string, value: string, unitType: string) {
   throw `Invalid Number: '${name}: ${value}' is not a valid number, cannot transform to '${unitType}' \n`;
 }
 
-function isNotBorderOrFixed(token: TransformedToken) {
-  return !token.path.some((path) => ["border", "fixed"].includes(path));
+function isStaticPx(token: TransformedToken) {
+  return !token.path.some((path) => ["base", "border", "fixed", "px"].includes(path));
 }
 
 function filter(token: TransformedToken, options: Config) {
   return (
     isSource(token) &&
     !isBreakpoint(token) &&
-    isNotBorderOrFixed(token) &&
+    isStaticPx(token) &&
     isDimension(token, options) &&
     !isFontRelated(token) &&
     !isCornerRadius(token) &&

packages/calcite-design-tokens/src/build/utils/dictionary.ts

@@ -1,9 +1,9 @@
-import { Dictionary, TransformedTokens } from "style-dictionary/types";
+import { Dictionary } from "style-dictionary/types";
 import { convertTokenData } from "style-dictionary/utils";
 
-export function fromTokens(tokens: TransformedTokens): Dictionary {
+export function fromTokens(tokens: Parameters<typeof convertTokenData>[0]): Dictionary {
   return {
-    tokens: tokens,
+    tokens: convertTokenData(tokens, { output: "object" }),
     allTokens: convertTokenData(tokens, { output: "array" }),
     tokenMap: convertTokenData(tokens, { output: "map" }),
   };

packages/calcite-design-tokens/src/build/utils/token-types.ts

@@ -47,12 +47,11 @@ export function isTypography(token: TransformedToken): boolean {
 
 interface IsThemedOptions {
   theme?: "light" | "dark";
-  targetPropName?: keyof Pick<TransformedToken, "filePath" | "path">;
 }
 
-export function isThemed(token: TransformedToken, options?: IsThemedOptions): boolean {
-  const { theme, targetPropName } = options || {};
-  const targetProp = token[targetPropName ?? "filePath"];
+export function isThemed(token: DesignToken, options?: IsThemedOptions): boolean {
+  const { theme } = options || {};
+  const targetProp = token.filePath;
 
   return (
     token.type === "color" &&

packages/calcite-design-tokens/src/config/calcite/dark.ts → packages/calcite-design-tokens/src/config/color/dark.ts

@@ -9,8 +9,8 @@ import { expandTypesMap as sdTypes } from "@tokens-studio/sd-transforms";
 import { transformers, filters, headers, formats } from "../../build/registry/index.js";
 
 const config: Config = {
-  source: ["src/tokens/semantic/calcite/dark.json"],
-  include: ["src/tokens/core/[!$]*.json"],
+  source: ["src/tokens/semantic/color/dark.json"],
+  include: ["src/tokens/semantic/color/dark.json", "src/tokens/core/[!$]*.json"],
   preprocessors: ["tokens-studio"],
   platforms: {
     scss: {

packages/calcite-design-tokens/src/config/calcite/light.ts → packages/calcite-design-tokens/src/config/color/light.ts

@@ -9,8 +9,8 @@ import { expandTypesMap as sdTypes } from "@tokens-studio/sd-transforms";
 import { transformers, filters, headers, formats } from "../../build/registry/index.js";
 
 const config: Config = {
-  source: ["src/tokens/semantic/calcite/light.json"],
-  include: ["src/tokens/core/[!$]*.json"],
+  source: ["src/tokens/semantic/color/light.json"],
+  include: ["src/tokens/semantic/color/light.json", "src/tokens/core/[!$]*.json"],
   preprocessors: ["tokens-studio"],
   platforms: {
     scss: {