From 43cfd4324b42ab598fc975ffae9e786df7194b15 Mon Sep 17 00:00:00 2001 From: Michael Mrowetz Date: Wed, 19 Jul 2023 10:42:09 -0400 Subject: [PATCH 1/6] feat: batch asset download to avoid upstream rate-limiting timeout (504) --- library/foundations/scripts/save-assets.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/library/foundations/scripts/save-assets.ts b/library/foundations/scripts/save-assets.ts index c7e0de69..c24217e9 100644 --- a/library/foundations/scripts/save-assets.ts +++ b/library/foundations/scripts/save-assets.ts @@ -64,6 +64,12 @@ const filterUnique = return acc; }; +// Promise to delay items in batches, e.g. to avoid upstream rate-limiting +const batchDelay = (requestIndex: number, batchSize = 20) => + new Promise((resolve) => { + setTimeout(() => resolve(), Math.floor(requestIndex / batchSize) * 20); + }); + loadLayersFile(snapshotFileName) .then(({ layers, order }) => { return order @@ -85,11 +91,13 @@ loadLayersFile(snapshotFileName) .then((assets) => // for every asset, download it and convert it to webp Promise.all( - assets.map(({ value, ...rest }) => - findAssetSizeAndDimensions(value).then((metadata) => ({ - ...rest, - ...metadata, - })) + assets.map(({ value, ...rest }, i) => + batchDelay(i) + .then(() => findAssetSizeAndDimensions(value)) + .then((metadata) => ({ + ...rest, + ...metadata, + })) ) ) ) From 0f45d873ab7d0756f368cd8bb97afdae85e18dbd Mon Sep 17 00:00:00 2001 From: Michael Mrowetz Date: Wed, 19 Jul 2023 10:43:06 -0400 Subject: [PATCH 2/6] perf: max size for downloaded assets --- library/foundations/scripts/save-assets.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/library/foundations/scripts/save-assets.ts b/library/foundations/scripts/save-assets.ts index c24217e9..38248cd6 100644 --- a/library/foundations/scripts/save-assets.ts +++ b/library/foundations/scripts/save-assets.ts @@ -40,6 +40,13 @@ const findAssetSizeAndDimensions = async (url: string) => { .then((response) => { console.info('downloaded. converting to webp'); return sharp(Buffer.from(response.data)) + .resize({ + // max width for now + width: 800, + height: 800, + fit: 'inside', + withoutEnlargement: true, + }) .webp() .toBuffer({ resolveWithObject: true }) .catch((e) => { From a7c63fb67a3bec1f2a0ccdb8d3e1931f58811d14 Mon Sep 17 00:00:00 2001 From: Michael Mrowetz Date: Wed, 19 Jul 2023 11:37:19 -0400 Subject: [PATCH 3/6] fix: increase max size --- library/foundations/scripts/save-assets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/foundations/scripts/save-assets.ts b/library/foundations/scripts/save-assets.ts index 38248cd6..5c6f3bb1 100644 --- a/library/foundations/scripts/save-assets.ts +++ b/library/foundations/scripts/save-assets.ts @@ -42,7 +42,7 @@ const findAssetSizeAndDimensions = async (url: string) => { return sharp(Buffer.from(response.data)) .resize({ // max width for now - width: 800, + width: 1200, height: 800, fit: 'inside', withoutEnlargement: true, From f801b97330f89d75d53aed003a857a9927f8708d Mon Sep 17 00:00:00 2001 From: Michael Mrowetz Date: Wed, 19 Jul 2023 16:35:17 -0400 Subject: [PATCH 4/6] feat: multi-sized images --- library/foundations/generator-config.ts | 5 ++ library/foundations/package.json | 3 +- library/foundations/scripts/save-assets.ts | 70 ++++++++++++------- .../foundations/src/helpers/src-to-src-set.ts | 14 ++++ .../image-text-item/image-text-item.tsx | 3 +- 5 files changed, 67 insertions(+), 28 deletions(-) create mode 100644 library/foundations/generator-config.ts create mode 100644 library/foundations/src/helpers/src-to-src-set.ts diff --git a/library/foundations/generator-config.ts b/library/foundations/generator-config.ts new file mode 100644 index 00000000..c44d9ac3 --- /dev/null +++ b/library/foundations/generator-config.ts @@ -0,0 +1,5 @@ +/** Customizable Configuration for the Generator */ +export const config = { + /** Responsive image size bounds (max width/ max height) */ + imageSizes: [400, 600, 800, 1000], +} as const; diff --git a/library/foundations/package.json b/library/foundations/package.json index 982636d3..99cb637f 100644 --- a/library/foundations/package.json +++ b/library/foundations/package.json @@ -6,7 +6,8 @@ "files": [ "docs/", "generated/*", - "src/" + "src/", + "./generator-config.ts" ], "types": "generated/design-tokens.types.ts", "repository": { diff --git a/library/foundations/scripts/save-assets.ts b/library/foundations/scripts/save-assets.ts index 5c6f3bb1..a93676ce 100644 --- a/library/foundations/scripts/save-assets.ts +++ b/library/foundations/scripts/save-assets.ts @@ -4,6 +4,7 @@ import axios from 'axios'; import { parseData } from './lib/token-parser'; import { TokenLayer, TokenLayers } from './lib/token-parser.types'; import { isVariableReference } from './lib/token-parser.utils'; +import { config } from '../generator-config'; const MEDIA_FILES_PATH = './generated'; @@ -37,25 +38,37 @@ const findAssetSizeAndDimensions = async (url: string) => { console.warn(`Error downloading ${url}`); throw e; }) - .then((response) => { + .then(async (response) => { console.info('downloaded. converting to webp'); - return sharp(Buffer.from(response.data)) - .resize({ - // max width for now - width: 1200, - height: 800, - fit: 'inside', - withoutEnlargement: true, - }) - .webp() - .toBuffer({ resolveWithObject: true }) - .catch((e) => { - console.warn(`Error parsing ${url}`); - throw e; - }); - }) - .then(({ data, info }) => { - return { data, info }; + const image = sharp(Buffer.from(response.data)); + + return Promise.all( + [...config.imageSizes, 'full' as const].map((width) => + image + .resize( + width === 'full' // full res version + ? undefined + : { + // resize to max-size, this might sometimes create duplicates, + // if the source image is small, but this makes the + width: width, + height: width, + fit: 'inside', + withoutEnlargement: true, + } + ) + .webp() + .toBuffer({ resolveWithObject: true }) + .then((v) => ({ + ...v, + width, + })) + .catch((e) => { + console.warn(`Error parsing ${url}`); + throw e; + }) + ) + ); }); }; @@ -98,15 +111,17 @@ loadLayersFile(snapshotFileName) .then((assets) => // for every asset, download it and convert it to webp Promise.all( - assets.map(({ value, ...rest }, i) => + assets.flatMap(({ value, ...rest }, i) => batchDelay(i) .then(() => findAssetSizeAndDimensions(value)) - .then((metadata) => ({ - ...rest, - ...metadata, - })) + .then((variations) => + variations.map((variation) => ({ + ...rest, + ...variation, + })) + ) ) - ) + ).then((v) => v.flat()) ) .then((downloadedAssets) => { // find all the directories we need to create @@ -148,13 +163,16 @@ const saveFiles = < data: Buffer; name: string; path: string; + width?: number | string; } >( downloadedAssets: T[] ): Promise => Promise.all( - downloadedAssets.map(({ name, data, path }) => { - const fileName = `${assetSavePath(path)}/${name}.webp`; + downloadedAssets.map(({ name, data, path, width }) => { + const fileName = `${assetSavePath(path)}/${name}${ + width && width !== 'full' ? `_w${width}` : '' + }.webp`; console.info(`SAVING ${fileName}`); return promises.writeFile(fileName, data); }) diff --git a/library/foundations/src/helpers/src-to-src-set.ts b/library/foundations/src/helpers/src-to-src-set.ts new file mode 100644 index 00000000..c9c3f62d --- /dev/null +++ b/library/foundations/src/helpers/src-to-src-set.ts @@ -0,0 +1,14 @@ +import { config } from '../../generator-config'; + +/** expands `src` to `srcSet` using the configured image sized */ +export const srcToSrcSet = (src: string) => { + const withoutExtension = src.replace('.webp', ''); + const srcSet = config.imageSizes + .map((s) => `${withoutExtension}_w${s}.webp ${s}w`) + .join(', '); + + return { + srcSet, + src, + }; +}; diff --git a/library/radius-examples/components/image-text-item/image-text-item.tsx b/library/radius-examples/components/image-text-item/image-text-item.tsx index b7a930c6..e91e9efd 100644 --- a/library/radius-examples/components/image-text-item/image-text-item.tsx +++ b/library/radius-examples/components/image-text-item/image-text-item.tsx @@ -1,6 +1,7 @@ import React, { forwardRef } from 'react'; import { cx } from '@emotion/css'; import { radiusTokens } from '@rangle/radius-foundations/generated/design-tokens.constants'; +import { srcToSrcSet } from '@rangle/radius-foundations/src/helpers/src-to-src-set'; import { RadiusAutoLayout, Typography, @@ -75,7 +76,7 @@ export const RadiusImageTextItem = forwardRef< > Date: Thu, 20 Jul 2023 16:02:53 -0400 Subject: [PATCH 5/6] fix: skip srcset for non-local files --- library/foundations/src/helpers/src-to-src-set.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/library/foundations/src/helpers/src-to-src-set.ts b/library/foundations/src/helpers/src-to-src-set.ts index c9c3f62d..29243fc6 100644 --- a/library/foundations/src/helpers/src-to-src-set.ts +++ b/library/foundations/src/helpers/src-to-src-set.ts @@ -2,6 +2,12 @@ import { config } from '../../generator-config'; /** expands `src` to `srcSet` using the configured image sized */ export const srcToSrcSet = (src: string) => { + const isLocalImage = src.startsWith('/') || src.startsWith('.'); + if (!isLocalImage) { + return { + src, + }; + } const withoutExtension = src.replace('.webp', ''); const srcSet = config.imageSizes .map((s) => `${withoutExtension}_w${s}.webp ${s}w`) From 2aa6bfce3bdf68dd070e5094ae43af2e1bc3008c Mon Sep 17 00:00:00 2001 From: Michael Mrowetz Date: Thu, 20 Jul 2023 16:17:21 -0400 Subject: [PATCH 6/6] docs: imporve assets docs --- library/foundations/README.md | 21 +++++++++++++-------- library/foundations/generator-config.ts | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/library/foundations/README.md b/library/foundations/README.md index 09bd6cba..d22d3700 100644 --- a/library/foundations/README.md +++ b/library/foundations/README.md @@ -20,29 +20,34 @@ This script is meant to be used locally to update the css variables in the theme ## yarn tokens:parse -This script parses the tokens and generates the token layers. It appends the package version as a suffix to the generated layers file if a RADIUS_LAYERS_SUFFIX environment variable is set. +This script parses the tokens and generates the token layers. It appends the package version as a suffix to the generated layers file if a `RADIUS_LAYERS_SUFFIX` environment variable is set. ## yarn tokens:validate -This script validates the token layers by comparing them with the original tokens file. It uses the generated layers file with the package version as a suffix if a RADIUS_LAYERS_SUFFIX environment variable is set. +This script validates the token layers by comparing them with the original tokens file. It uses the generated layers file with the package version as a suffix if a `RADIUS_LAYERS_SUFFIX` environment variable is set. The validator identifies **four** types of contexts/layers: + - **Source layers (`core`)** — tokens there are not supposed to be used directly in the code, only as reference to others. they need no special treatment — the exporter detects them automatically and resolves their variables to values during the build - **Semantic layers** are divided in two specialized categories: - **Selectable layers** (`mode-light` or `mode-dark`) apply to some or all website features in a way developers can select using a className. To identify one of these, the exporter looks for a token named `section-name`. The value is the name of the scenario for each mode (ex.: `Light Mode` or `High Constrast`) that will be normalized automatically by the exporter to become a className (ex.: light-mode or high-contrast). The important thing to know about selectable layers is that they need to have the same tokens — or the exporter will throw an error. - - **Viewport size layers** (`breakpoint-large`, etc) apply automatically to semantic situations that vary based on the viewport size (mobile vs desktop, for example). In order for this to work, they should have special tokens `screen-min-size` and `screen-max-size` - to indicate their tokens are active when the viewport is between these values. Like selectable tokens, they also need to have the same tokens — or the exporter will throw an error. -- **Consolidation layers** (`components`) will only contain references to other layers. If these references come from more than one semantic layer (for example, if a token called `component.button.background` points to a token `semantic.surface.action` that exists in both `mode-light` and `mode-dark`) the exporter will take note and make sure the selection is properly controlled by the changes in the semantic layers (for example, in CSS, adding all the necessary classes to the `@radius-components` CSS Layer). Consolidation layers do not need special variables, like ‘screen-min-size’ or ’section-name). The exporter detects them automatically + - **Viewport size layers** (`breakpoint-large`, etc) apply automatically to semantic situations that vary based on the viewport size (mobile vs desktop, for example). In order for this to work, they should have special tokens `screen-min-size` and `screen-max-size` - to indicate their tokens are active when the viewport is between these values. Like selectable tokens, they also need to have the same tokens — or the exporter will throw an error. +- **Consolidation layers** (`components`) will only contain references to other layers. If these references come from more than one semantic layer (for example, if a token called `component.button.background` points to a token `semantic.surface.action` that exists in both `mode-light` and `mode-dark`) the exporter will take note and make sure the selection is properly controlled by the changes in the semantic layers (for example, in CSS, adding all the necessary classes to the `@radius-components` CSS Layer). Consolidation layers do not need special variables, like `screen-min-size` or `section-name`). The exporter detects them automatically + +## yarn tokens:assets + +Downloads all assets from snapshot file provided and resizes them to the bounds configured via `imageSizes` in `generator-config.ts`. Output assets are stored in `generated/brand//assets/` ## yarn tokens:generate -This script generates the theme by using the generated token layers file with the package version as a suffix if a RADIUS_LAYERS_SUFFIX environment variable is set. It outputs the theme to the generated directory in the format specified by the second argument (in this case, css). +This script generates the theme by using the generated token layers file with the package version as a suffix if a `RADIUS_LAYERS_SUFFIX` environment variable is set. It outputs the theme to the generated directory in the format specified by the second argument (in this case, css). -It is important to note that the tokens:ci and tokens:update scripts use the RADIUS_LAYERS_SUFFIX environment variable to append the package version to the output file name. It is recommended to use these scripts in the specific scenarios described above. +It is important to note that the tokens:ci and tokens:update scripts use the `RADIUS_LAYERS_SUFFIX` environment variable to append the package version to the output file name. It is recommended to use these scripts in the specific scenarios described above. ## yarn tokens:types This script generates type definition files by using the generated token layers file. It outputs them to the generated directory as a typescript file. It will run prettier afterwards to ensure the file obeys linting rules. -These type definitions allow developers to use tokens in their component files without resorting to -javascript themes in runtime. \ No newline at end of file +These type definitions allow developers to use tokens in their component files without resorting to +javascript themes in runtime. diff --git a/library/foundations/generator-config.ts b/library/foundations/generator-config.ts index c44d9ac3..ae2ccee2 100644 --- a/library/foundations/generator-config.ts +++ b/library/foundations/generator-config.ts @@ -1,5 +1,5 @@ /** Customizable Configuration for the Generator */ export const config = { - /** Responsive image size bounds (max width/ max height) */ + /** Responsive image size bounds (max width/ max height in px) */ imageSizes: [400, 600, 800, 1000], } as const;