Skip to main content
EVOKORE// BROWSE
>

./browse/prompts

15 NODES
🤖system prompt•7 months ago

nft-standards

Implement NFT standards (ERC-721, ERC-1155) with proper metadata

coding
⭐1
# NFT Standards Master ERC-721 and ERC-1155 NFT standards, metadata best practices, and advanced NFT features. ## When to Use This Skill - Creating NFT collections (art, gaming, collectibles) - Implementing marketplace functionality - Building on-chain or off-chain metadata - Creating soulbound tokens (non-transferable) - Implementing royalties and revenue sharing - Developing dynamic/evolving NFTs ## ERC-721 (Non-Fungible Token Standard) ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC721/extensions/ERC721URIStorage.sol"; import "@openzeppelin/contracts/token/ERC721/extensions/ERC721Enumerable.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; import "@openzeppelin/contracts/utils/Counters.sol"; contract MyNFT is ERC721URIStorage, ERC721Enumerable, Ownable { using Counters for Counters.Counter; Counters.Counter private _tokenIds; uint256 public constant MAX_SUPPLY = 10000; uint256 public constant MINT_PRICE = 0.08 ether; uint256 public constant MAX_PER_MINT = 20; constructor() ERC721("MyNFT", "MNFT") {} function mint(uint256 quantity) external payable { require(quantity > 0 && quantity <= MAX_PER_MINT, "Invalid quantity"); require(_tokenIds.current() + quantity <= MAX_SUPPLY, "Exceeds max supply"); require(msg.value >= MINT_PRICE * quantity, "Insufficient payment"); for (uint256 i = 0; i < quantity; i++) { _tokenIds.increment(); uint256 newTokenId = _tokenIds.current(); _safeMint(msg.sender, newTokenId); _setTokenURI(newTokenId, generateTokenURI(newTokenId)); } } function generateTokenURI(uint256 tokenId) internal pure returns (string memory) { // Return IPFS URI or on-chain metadata return string(abi.encodePacked("ipfs://QmHash/", Strings.toString(tokenId), ".json")); } // Required overrides function _beforeTokenTransfer( address from, address to, uint256 tokenId, uint256 batchSize ) internal override(ERC721, ERC721Enumerable) { super._beforeTokenTransfer(from, to, tokenId, batchSize); } function _burn(uint256 tokenId) internal override(ERC721, ERC721URIStorage) { super._burn(tokenId); } function tokenURI(uint256 tokenId) public view override(ERC721, ERC721URIStorage) returns (string memory) { return super.tokenURI(tokenId); } function supportsInterface(bytes4 interfaceId) public view override(ERC721, ERC721Enumerable) returns (bool) { return super.supportsInterface(interfaceId); } function withdraw() external onlyOwner { payable(owner()).transfer(address(this).balance); } } ``` ## ERC-1155 (Multi-Token Standard) ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC1155/ERC1155.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; contract GameItems is ERC1155, Ownable { uint256 public constant SWORD = 1; uint256 public constant SHIELD = 2; uint256 public constant POTION = 3; mapping(uint256 => uint256) public tokenSupply; mapping(uint256 => uint256) public maxSupply; constructor() ERC1155("ipfs://QmBaseHash/{id}.json") { maxSupply[SWORD] = 1000; maxSupply[SHIELD] = 500; maxSupply[POTION] = 10000; } function mint( address to, uint256 id, uint256 amount ) external onlyOwner { require(tokenSupply[id] + amount <= maxSupply[id], "Exceeds max supply"); _mint(to, id, amount, ""); tokenSupply[id] += amount; } function mintBatch( address to, uint256[] memory ids, uint256[] memory amounts ) external onlyOwner { for (uint256 i = 0; i < ids.length; i++) { require(tokenSupply[ids[i]] + amounts[i] <= maxSupply[ids[i]], "Exceeds max supply"); tokenSupply[ids[i]] += amounts[i]; } _mintBatch(to, ids, amounts, ""); } function burn( address from, uint256 id, uint256 amount ) external { require(from == msg.sender || isApprovedForAll(from, msg.sender), "Not authorized"); _burn(from, id, amount); tokenSupply[id] -= amount; } } ``` ## Metadata Standards ### Off-Chain Metadata (IPFS) ```json { "name": "NFT #1", "description": "Description of the NFT", "image": "ipfs://QmImageHash", "attributes": [ { "trait_type": "Background", "value": "Blue" }, { "trait_type": "Rarity", "value": "Legendary" }, { "trait_type": "Power", "value": 95, "display_type": "number", "max_value": 100 } ] } ``` ### On-Chain Metadata ```solidity contract OnChainNFT is ERC721 { struct Traits { uint8 background; uint8 body; uint8 head; uint8 rarity; } mapping(uint256 => Traits) public tokenTraits; function tokenURI(uint256 tokenId) public view override returns (string memory) { Traits memory traits = tokenTraits[tokenId]; string memory json = Base64.encode( bytes( string( abi.encodePacked( '{"name": "NFT #', Strings.toString(tokenId), '",', '"description": "On-chain NFT",', '"image": "data:image/svg+xml;base64,', generateSVG(traits), '",', '"attributes": [', '{"trait_type": "Background", "value": "', Strings.toString(traits.background), '"},', '{"trait_type": "Rarity", "value": "', getRarityName(traits.rarity), '"}', ']}' ) ) ) ); return string(abi.encodePacked("data:application/json;base64,", json)); } function generateSVG(Traits memory traits) internal pure returns (string memory) { // Generate SVG based on traits return "..."; } } ``` ## Royalties (EIP-2981) ```solidity import "@openzeppelin/contracts/interfaces/IERC2981.sol"; contract NFTWithRoyalties is ERC721, IERC2981 { address public royaltyRecipient; uint96 public royaltyFee = 500; // 5% constructor() ERC721("Royalty NFT", "RNFT") { royaltyRecipient = msg.sender; } function royaltyInfo(uint256 tokenId, uint256 salePrice) external view override returns (address receiver, uint256 royaltyAmount) { return (royaltyRecipient, (salePrice * royaltyFee) / 10000); } function setRoyalty(address recipient, uint96 fee) external onlyOwner { require(fee <= 1000, "Royalty fee too high"); // Max 10% royaltyRecipient = recipient; royaltyFee = fee; } function supportsInterface(bytes4 interfaceId) public view override(ERC721, IERC165) returns (bool) { return interfaceId == type(IERC2981).interfaceId || super.supportsInterface(interfaceId); } } ``` ## Soulbound Tokens (Non-Transferable) ```solidity contract SoulboundToken is ERC721 { constructor() ERC721("Soulbound", "SBT") {} function _beforeTokenTransfer( address from, address to, uint256 tokenId, uint256 batchSize ) internal virtual override { require(from == address(0) || to == address(0), "Token is soulbound"); super._beforeTokenTransfer(from, to, tokenId, batchSize); } function mint(address to) external { uint256 tokenId = totalSupply() + 1; _safeMint(to, tokenId); } // Burn is allowed (user can destroy their SBT) function burn(uint256 tokenId) external { require(ownerOf(tokenId) == msg.sender, "Not token owner"); _burn(tokenId); } } ``` ## Dynamic NFTs ```solidity contract DynamicNFT is ERC721 { struct TokenState { uint256 level; uint256 experience; uint256 lastUpdated; } mapping(uint256 => TokenState) public tokenStates; function gainExperience(uint256 tokenId, uint256 exp) external { require(ownerOf(tokenId) == msg.sender, "Not token owner"); TokenState storage state = tokenStates[tokenId]; state.experience += exp; // Level up logic if (state.experience >= state.level * 100) { state.level++; } state.lastUpdated = block.timestamp; } function tokenURI(uint256 tokenId) public view override returns (string memory) { TokenState memory state = tokenStates[tokenId]; // Generate metadata based on current state return generateMetadata(tokenId, state); } function generateMetadata(uint256 tokenId, TokenState memory state) internal pure returns (string memory) { // Dynamic metadata generation return ""; } } ``` ## Gas-Optimized Minting (ERC721A) ```solidity import "erc721a/contracts/ERC721A.sol"; contract OptimizedNFT is ERC721A { uint256 public constant MAX_SUPPLY = 10000; uint256 public constant MINT_PRICE = 0.05 ether; constructor() ERC721A("Optimized NFT", "ONFT") {} function mint(uint256 quantity) external payable { require(_totalMinted() + quantity <= MAX_SUPPLY, "Exceeds max supply"); require(msg.value >= MINT_PRICE * quantity, "Insufficient payment"); _mint(msg.sender, quantity); } function _baseURI() internal pure override returns (string memory) { return "ipfs://QmBaseHash/"; } } ``` ## Resources - **references/erc721.md**: ERC-721 specification details - **references/erc1155.md**: ERC-1155 multi-token standard - **references/metadata-standards.md**: Metadata best practices - **references/enumeration.md**: Token enumeration patterns - **assets/erc721-contract.sol**: Production ERC-721 template - **assets/erc1155-contract.sol**: Production ERC-1155 template - **assets/metadata-schema.json**: Standard metadata format - **assets/metadata-uploader.py**: IPFS upload utility ## Best Practices 1. **Use OpenZeppelin**: Battle-tested implementations 2. **Pin Metadata**: Use IPFS with pinning service 3. **Implement Royalties**: EIP-2981 for marketplace compatibility 4. **Gas Optimization**: Use ERC721A for batch minting 5. **Reveal Mechanism**: Placeholder → reveal pattern 6. **Enumeration**: Support walletOfOwner for marketplaces 7. **Whitelist**: Merkle trees for efficient whitelisting ## Marketplace Integration - OpenSea: ERC-721/1155, metadata standards - LooksRare: Royalty enforcement - Rarible: Protocol fees, lazy minting - Blur: Gas-optimized trading
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

postgresql-table-design

Design a PostgreSQL-specific schema. Covers best-practices, data

data
⭐1
# PostgreSQL Table Design ## Core Rules - Define a **PRIMARY KEY** for reference tables (users, orders, etc.). Not always needed for time-series/event/log data. When used, prefer `BIGINT GENERATED ALWAYS AS IDENTITY`; use `UUID` only when global uniqueness/opacity is needed. - **Normalize first (to 3NF)** to eliminate data redundancy and update anomalies; denormalize **only** for measured, high-ROI reads where join performance is proven problematic. Premature denormalization creates maintenance burden. - Add **NOT NULL** everywhere it’s semantically required; use **DEFAULT**s for common values. - Create **indexes for access paths you actually query**: PK/unique (auto), **FK columns (manual!)**, frequent filters/sorts, and join keys. - Prefer **TIMESTAMPTZ** for event time; **NUMERIC** for money; **TEXT** for strings; **BIGINT** for integer values, **DOUBLE PRECISION** for floats (or `NUMERIC` for exact decimal arithmetic). ## PostgreSQL “Gotchas” - **Identifiers**: unquoted → lowercased. Avoid quoted/mixed-case names. Convention: use `snake_case` for table/column names. - **Unique + NULLs**: UNIQUE allows multiple NULLs. Use `UNIQUE (...) NULLS NOT DISTINCT` (PG15+) to restrict to one NULL. - **FK indexes**: PostgreSQL **does not** auto-index FK columns. Add them. - **No silent coercions**: length/precision overflows error out (no truncation). Example: inserting 999 into `NUMERIC(2,0)` fails with error, unlike some databases that silently truncate or round. - **Sequences/identity have gaps** (normal; don't "fix"). Rollbacks, crashes, and concurrent transactions create gaps in ID sequences (1, 2, 5, 6...). This is expected behavior—don't try to make IDs consecutive. - **Heap storage**: no clustered PK by default (unlike SQL Server/MySQL InnoDB); `CLUSTER` is one-off reorganization, not maintained on subsequent inserts. Row order on disk is insertion order unless explicitly clustered. - **MVCC**: updates/deletes leave dead tuples; vacuum handles them—design to avoid hot wide-row churn. ## Data Types - **IDs**: `BIGINT GENERATED ALWAYS AS IDENTITY` preferred (`GENERATED BY DEFAULT` also fine); `UUID` when merging/federating/used in a distributed system or for opaque IDs. Generate with `uuidv7()` (preferred if using PG18+) or `gen_random_uuid()` (if using an older PG version). - **Integers**: prefer `BIGINT` unless storage space is critical; `INTEGER` for smaller ranges; avoid `SMALLINT` unless constrained. - **Floats**: prefer `DOUBLE PRECISION` over `REAL` unless storage space is critical. Use `NUMERIC` for exact decimal arithmetic. - **Strings**: prefer `TEXT`; if length limits needed, use `CHECK (LENGTH(col) <= n)` instead of `VARCHAR(n)`; avoid `CHAR(n)`. Use `BYTEA` for binary data. Large strings/binary (>2KB default threshold) automatically stored in TOAST with compression. TOAST storage: `PLAIN` (no TOAST), `EXTENDED` (compress + out-of-line), `EXTERNAL` (out-of-line, no compress), `MAIN` (compress, keep in-line if possible). Default `EXTENDED` usually optimal. Control with `ALTER TABLE tbl ALTER COLUMN col SET STORAGE strategy` and `ALTER TABLE tbl SET (toast_tuple_target = 4096)` for threshold. Case-insensitive: for locale/accent handling use non-deterministic collations; for plain ASCII use expression indexes on `LOWER(col)` (preferred unless column needs case-insensitive PK/FK/UNIQUE) or `CITEXT`. - **Money**: `NUMERIC(p,s)` (never float). - **Time**: `TIMESTAMPTZ` for timestamps; `DATE` for date-only; `INTERVAL` for durations. Avoid `TIMESTAMP` (without timezone). Use `now()` for transaction start time, `clock_timestamp()` for current wall-clock time. - **Booleans**: `BOOLEAN` with `NOT NULL` constraint unless tri-state values are required. - **Enums**: `CREATE TYPE ... AS ENUM` for small, stable sets (e.g. US states, days of week). For business-logic-driven and evolving values (e.g. order statuses) → use TEXT (or INT) + CHECK or lookup table. - **Arrays**: `TEXT[]`, `INTEGER[]`, etc. Use for ordered lists where you query elements. Index with **GIN** for containment (`@>`, `<@`) and overlap (`&&`) queries. Access: `arr[1]` (1-indexed), `arr[1:3]` (slicing). Good for tags, categories; avoid for relations—use junction tables instead. Literal syntax: `'{val1,val2}'` or `ARRAY[val1,val2]`. - **Range types**: `daterange`, `numrange`, `tstzrange` for intervals. Support overlap (`&&`), containment (`@>`), operators. Index with **GiST**. Good for scheduling, versioning, numeric ranges. Pick a bounds scheme and use it consistently; prefer `[)` (inclusive/exclusive) by default. - **Network types**: `INET` for IP addresses, `CIDR` for network ranges, `MACADDR` for MAC addresses. Support network operators (`<<`, `>>`, `&&`). - **Geometric types**: `POINT`, `LINE`, `POLYGON`, `CIRCLE` for 2D spatial data. Index with **GiST**. Consider **PostGIS** for advanced spatial features. - **Text search**: `TSVECTOR` for full-text search documents, `TSQUERY` for search queries. Index `tsvector` with **GIN**. Always specify language: `to_tsvector('english', col)` and `to_tsquery('english', 'query')`. Never use single-argument versions. This applies to both index expressions and queries. - **Domain types**: `CREATE DOMAIN email AS TEXT CHECK (VALUE ~ '^[^@]+@[^@]+$')` for reusable custom types with validation. Enforces constraints across tables. - **Composite types**: `CREATE TYPE address AS (street TEXT, city TEXT, zip TEXT)` for structured data within columns. Access with `(col).field` syntax. - **JSONB**: preferred over JSON; index with **GIN**. Use only for optional/semi-structured attrs. ONLY use JSON if the original ordering of the contents MUST be preserved. - **Vector types**: `vector` type by `pgvector` for vector similarity search for embeddings. ### Do not use the following data types - DO NOT use `timestamp` (without time zone); DO use `timestamptz` instead. - DO NOT use `char(n)` or `varchar(n)`; DO use `text` instead. - DO NOT use `money` type; DO use `numeric` instead. - DO NOT use `timetz` type; DO use `timestamptz` instead. - DO NOT use `timestamptz(0)` or any other precision specification; DO use `timestamptz` instead - DO NOT use `serial` type; DO use `generated always as identity` instead. ## Table Types - **Regular**: default; fully durable, logged. - **TEMPORARY**: session-scoped, auto-dropped, not logged. Faster for scratch work. - **UNLOGGED**: persistent but not crash-safe. Faster writes; good for caches/staging. ## Row-Level Security Enable with `ALTER TABLE tbl ENABLE ROW LEVEL SECURITY`. Create policies: `CREATE POLICY user_access ON orders FOR SELECT TO app_users USING (user_id = current_user_id())`. Built-in user-based access control at the row level. ## Constraints - **PK**: implicit UNIQUE + NOT NULL; creates a B-tree index. - **FK**: specify `ON DELETE/UPDATE` action (`CASCADE`, `RESTRICT`, `SET NULL`, `SET DEFAULT`). Add explicit index on referencing column—speeds up joins and prevents locking issues on parent deletes/updates. Use `DEFERRABLE INITIALLY DEFERRED` for circular FK dependencies checked at transaction end. - **UNIQUE**: creates a B-tree index; allows multiple NULLs unless `NULLS NOT DISTINCT` (PG15+). Standard behavior: `(1, NULL)` and `(1, NULL)` are allowed. With `NULLS NOT DISTINCT`: only one `(1, NULL)` allowed. Prefer `NULLS NOT DISTINCT` unless you specifically need duplicate NULLs. - **CHECK**: row-local constraints; NULL values pass the check (three-valued logic). Example: `CHECK (price > 0)` allows NULL prices. Combine with `NOT NULL` to enforce: `price NUMERIC NOT NULL CHECK (price > 0)`. - **EXCLUDE**: prevents overlapping values using operators. `EXCLUDE USING gist (room_id WITH =, booking_period WITH &&)` prevents double-booking rooms. Requires appropriate index type (often GiST). ## Indexing - **B-tree**: default for equality/range queries (`=`, `<`, `>`, `BETWEEN`, `ORDER BY`) - **Composite**: order matters—index used if equality on leftmost prefix (`WHERE a = ? AND b > ?` uses index on `(a,b)`, but `WHERE b = ?` does not). Put most selective/frequently filtered columns first. - **Covering**: `CREATE INDEX ON tbl (id) INCLUDE (name, email)` - includes non-key columns for index-only scans without visiting table. - **Partial**: for hot subsets (`WHERE status = 'active'` → `CREATE INDEX ON tbl (user_id) WHERE status = 'active'`). Any query with `status = 'active'` can use this index. - **Expression**: for computed search keys (`CREATE INDEX ON tbl (LOWER(email))`). Expression must match exactly in WHERE clause: `WHERE LOWER(email) = 'user@example.com'`. - **GIN**: JSONB containment/existence, arrays (`@>`, `?`), full-text search (`@@`) - **GiST**: ranges, geometry, exclusion constraints - **BRIN**: very large, naturally ordered data (time-series)—minimal storage overhead. Effective when row order on disk correlates with indexed column (insertion order or after `CLUSTER`). ## Partitioning - Use for very large tables (>100M rows) where queries consistently filter on partition key (often time/date). - Alternate use: use for tables where data maintenance tasks dictates e.g. data pruned or bulk replaced periodically - **RANGE**: common for time-series (`PARTITION BY RANGE (created_at)`). Create partitions: `CREATE TABLE logs_2024_01 PARTITION OF logs FOR VALUES FROM ('2024-01-01') TO ('2024-02-01')`. **TimescaleDB** automates time-based or ID-based partitioning with retention policies and compression. - **LIST**: for discrete values (`PARTITION BY LIST (region)`). Example: `FOR VALUES IN ('us-east', 'us-west')`. - **HASH**: for even distribution when no natural key (`PARTITION BY HASH (user_id)`). Creates N partitions with modulus. - **Constraint exclusion**: requires `CHECK` constraints on partitions for query planner to prune. Auto-created for declarative partitioning (PG10+). - Prefer declarative partitioning or hypertables. Do NOT use table inheritance. - **Limitations**: no global UNIQUE constraints—include partition key in PK/UNIQUE. FKs from partitioned tables not supported; use triggers. ## Special Considerations ### Update-Heavy Tables - **Separate hot/cold columns**—put frequently updated columns in separate table to minimize bloat. - **Use `fillfactor=90`** to leave space for HOT updates that avoid index maintenance. - **Avoid updating indexed columns**—prevents beneficial HOT updates. - **Partition by update patterns**—separate frequently updated rows in a different partition from stable data. ### Insert-Heavy Workloads - **Minimize indexes**—only create what you query; every index slows inserts. - **Use `COPY` or multi-row `INSERT`** instead of single-row inserts. - **UNLOGGED tables** for rebuildable staging data—much faster writes. - **Defer index creation** for bulk loads—>drop index, load data, recreate indexes. - **Partition by time/hash** to distribute load. **TimescaleDB** automates partitioning and compression of insert-heavy data. - **Use a natural key for primary key** such as a (timestamp, device_id) if enforcing global uniqueness is important many insert-heavy tables don't need a primary key at all. - If you do need a surrogate key, **Prefer `BIGINT GENERATED ALWAYS AS IDENTITY` over `UUID`**. ### Upsert-Friendly Design - **Requires UNIQUE index** on conflict target columns—`ON CONFLICT (col1, col2)` needs exact matching unique index (partial indexes don't work). - **Use `EXCLUDED.column`** to reference would-be-inserted values; only update columns that actually changed to reduce write overhead. - **`DO NOTHING` faster** than `DO UPDATE` when no actual update needed. ### Safe Schema Evolution - **Transactional DDL**: most DDL operations can run in transactions and be rolled back—`BEGIN; ALTER TABLE...; ROLLBACK;` for safe testing. - **Concurrent index creation**: `CREATE INDEX CONCURRENTLY` avoids blocking writes but can't run in transactions. - **Volatile defaults cause rewrites**: adding `NOT NULL` columns with volatile defaults (e.g., `now()`, `gen_random_uuid()`) rewrites entire table. Non-volatile defaults are fast. - **Drop constraints before columns**: `ALTER TABLE DROP CONSTRAINT` then `DROP COLUMN` to avoid dependency issues. - **Function signature changes**: `CREATE OR REPLACE` with different arguments creates overloads, not replacements. DROP old version if no overload desired. ## Generated Columns - `... GENERATED ALWAYS AS (<expr>) STORED` for computed, indexable fields. PG18+ adds `VIRTUAL` columns (computed on read, not stored). ## Extensions - **`pgcrypto`**: `crypt()` for password hashing. - **`uuid-ossp`**: alternative UUID functions; prefer `pgcrypto` for new projects. - **`pg_trgm`**: fuzzy text search with `%` operator, `similarity()` function. Index with GIN for `LIKE '%pattern%'` acceleration. - **`citext`**: case-insensitive text type. Prefer expression indexes on `LOWER(col)` unless you need case-insensitive constraints. - **`btree_gin`/`btree_gist`**: enable mixed-type indexes (e.g., GIN index on both JSONB and text columns). - **`hstore`**: key-value pairs; mostly superseded by JSONB but useful for simple string mappings. - **`timescaledb`**: essential for time-series—automated partitioning, retention, compression, continuous aggregates. - **`postgis`**: comprehensive geospatial support beyond basic geometric types—essential for location-based applications. - **`pgvector`**: vector similarity search for embeddings. - **`pgaudit`**: audit logging for all database activity. ## JSONB Guidance - Prefer `JSONB` with **GIN** index. - Default: `CREATE INDEX ON tbl USING GIN (jsonb_col);` → accelerates: - **Containment** `jsonb_col @> '{"k":"v"}'` - **Key existence** `jsonb_col ? 'k'`, **any/all keys** `?\|`, `?&` - **Path containment** on nested docs - **Disjunction** `jsonb_col @> ANY(ARRAY['{"status":"active"}', '{"status":"pending"}'])` - Heavy `@>` workloads: consider opclass `jsonb_path_ops` for smaller/faster containment-only indexes: - `CREATE INDEX ON tbl USING GIN (jsonb_col jsonb_path_ops);` - **Trade-off**: loses support for key existence (`?`, `?|`, `?&`) queries—only supports containment (`@>`) - Equality/range on a specific scalar field: extract and index with B-tree (generated column or expression): - `ALTER TABLE tbl ADD COLUMN price INT GENERATED ALWAYS AS ((jsonb_col->>'price')::INT) STORED;` - `CREATE INDEX ON tbl (price);` - Prefer queries like `WHERE price BETWEEN 100 AND 500` (uses B-tree) over `WHERE (jsonb_col->>'price')::INT BETWEEN 100 AND 500` without index. - Arrays inside JSONB: use GIN + `@>` for containment (e.g., tags). Consider `jsonb_path_ops` if only doing containment. - Keep core relations in tables; use JSONB for optional/variable attributes. - Use constraints to limit allowed JSONB values in a column e.g. `config JSONB NOT NULL CHECK(jsonb_typeof(config) = 'object')` ## Examples ### Users ```sql CREATE TABLE users ( user_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, email TEXT NOT NULL UNIQUE, name TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE UNIQUE INDEX ON users (LOWER(email)); CREATE INDEX ON users (created_at); ``` ### Orders ```sql CREATE TABLE orders ( order_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, user_id BIGINT NOT NULL REFERENCES users(user_id), status TEXT NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING','PAID','CANCELED')), total NUMERIC(10,2) NOT NULL CHECK (total > 0), created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE INDEX ON orders (user_id); CREATE INDEX ON orders (created_at); ``` ### JSONB ```sql CREATE TABLE profiles ( user_id BIGINT PRIMARY KEY REFERENCES users(user_id), attrs JSONB NOT NULL DEFAULT '{}', theme TEXT GENERATED ALWAYS AS (attrs->>'theme') STORED ); CREATE INDEX profiles_attrs_gin ON profiles USING GIN (attrs); ```
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

nx-workspace-patterns

Configure and optimize Nx monorepo workspaces. Use when setting up

coding
⭐1
# Nx Workspace Patterns Production patterns for Nx monorepo management. ## When to Use This Skill - Setting up new Nx workspaces - Configuring project boundaries - Optimizing CI with affected commands - Implementing remote caching - Managing dependencies between projects - Migrating to Nx ## Core Concepts ### 1. Nx Architecture ``` workspace/ ├── apps/ # Deployable applications │ ├── web/ │ └── api/ ├── libs/ # Shared libraries │ ├── shared/ │ │ ├── ui/ │ │ └── utils/ │ └── feature/ │ ├── auth/ │ └── dashboard/ ├── tools/ # Custom executors/generators ├── nx.json # Nx configuration └── workspace.json # Project configuration ``` ### 2. Library Types | Type | Purpose | Example | | --------------- | -------------------------------- | ------------------- | | **feature** | Smart components, business logic | `feature-auth` | | **ui** | Presentational components | `ui-buttons` | | **data-access** | API calls, state management | `data-access-users` | | **util** | Pure functions, helpers | `util-formatting` | | **shell** | App bootstrapping | `shell-web` | ## Templates ### Template 1: nx.json Configuration ```json { "$schema": "./node_modules/nx/schemas/nx-schema.json", "npmScope": "myorg", "affected": { "defaultBase": "main" }, "tasksRunnerOptions": { "default": { "runner": "nx/tasks-runners/default", "options": { "cacheableOperations": [ "build", "lint", "test", "e2e", "build-storybook" ], "parallel": 3 } } }, "targetDefaults": { "build": { "dependsOn": ["^build"], "inputs": ["production", "^production"], "cache": true }, "test": { "inputs": ["default", "^production", "{workspaceRoot}/jest.preset.js"], "cache": true }, "lint": { "inputs": ["default", "{workspaceRoot}/.eslintrc.json"], "cache": true }, "e2e": { "inputs": ["default", "^production"], "cache": true } }, "namedInputs": { "default": ["{projectRoot}/**/*", "sharedGlobals"], "production": [ "default", "!{projectRoot}/**/?(*.)+(spec|test).[jt]s?(x)?(.snap)", "!{projectRoot}/tsconfig.spec.json", "!{projectRoot}/jest.config.[jt]s", "!{projectRoot}/.eslintrc.json" ], "sharedGlobals": [ "{workspaceRoot}/babel.config.json", "{workspaceRoot}/tsconfig.base.json" ] }, "generators": { "@nx/react": { "application": { "style": "css", "linter": "eslint", "bundler": "webpack" }, "library": { "style": "css", "linter": "eslint" }, "component": { "style": "css" } } } } ``` ### Template 2: Project Configuration ```json // apps/web/project.json { "name": "web", "$schema": "../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "apps/web/src", "projectType": "application", "tags": ["type:app", "scope:web"], "targets": { "build": { "executor": "@nx/webpack:webpack", "outputs": ["{options.outputPath}"], "defaultConfiguration": "production", "options": { "compiler": "babel", "outputPath": "dist/apps/web", "index": "apps/web/src/index.html", "main": "apps/web/src/main.tsx", "tsConfig": "apps/web/tsconfig.app.json", "assets": ["apps/web/src/assets"], "styles": ["apps/web/src/styles.css"] }, "configurations": { "development": { "extractLicenses": false, "optimization": false, "sourceMap": true }, "production": { "optimization": true, "outputHashing": "all", "sourceMap": false, "extractLicenses": true } } }, "serve": { "executor": "@nx/webpack:dev-server", "defaultConfiguration": "development", "options": { "buildTarget": "web:build" }, "configurations": { "development": { "buildTarget": "web:build:development" }, "production": { "buildTarget": "web:build:production" } } }, "test": { "executor": "@nx/jest:jest", "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], "options": { "jestConfig": "apps/web/jest.config.ts", "passWithNoTests": true } }, "lint": { "executor": "@nx/eslint:lint", "outputs": ["{options.outputFile}"], "options": { "lintFilePatterns": ["apps/web/**/*.{ts,tsx,js,jsx}"] } } } } ``` ### Template 3: Module Boundary Rules ```json // .eslintrc.json { "root": true, "ignorePatterns": ["**/*"], "plugins": ["@nx"], "overrides": [ { "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], "rules": { "@nx/enforce-module-boundaries": [ "error", { "enforceBuildableLibDependency": true, "allow": [], "depConstraints": [ { "sourceTag": "type:app", "onlyDependOnLibsWithTags": [ "type:feature", "type:ui", "type:data-access", "type:util" ] }, { "sourceTag": "type:feature", "onlyDependOnLibsWithTags": [ "type:ui", "type:data-access", "type:util" ] }, { "sourceTag": "type:ui", "onlyDependOnLibsWithTags": ["type:ui", "type:util"] }, { "sourceTag": "type:data-access", "onlyDependOnLibsWithTags": ["type:data-access", "type:util"] }, { "sourceTag": "type:util", "onlyDependOnLibsWithTags": ["type:util"] }, { "sourceTag": "scope:web", "onlyDependOnLibsWithTags": ["scope:web", "scope:shared"] }, { "sourceTag": "scope:api", "onlyDependOnLibsWithTags": ["scope:api", "scope:shared"] }, { "sourceTag": "scope:shared", "onlyDependOnLibsWithTags": ["scope:shared"] } ] } ] } } ] } ``` ### Template 4: Custom Generator ```typescript // tools/generators/feature-lib/index.ts import { Tree, formatFiles, generateFiles, joinPathFragments, names, readProjectConfiguration, } from "@nx/devkit"; import { libraryGenerator } from "@nx/react"; interface FeatureLibraryGeneratorSchema { name: string; scope: string; directory?: string; } export default async function featureLibraryGenerator( tree: Tree, options: FeatureLibraryGeneratorSchema, ) { const { name, scope, directory } = options; const projectDirectory = directory ? `${directory}/${name}` : `libs/${scope}/feature-${name}`; // Generate base library await libraryGenerator(tree, { name: `feature-${name}`, directory: projectDirectory, tags: `type:feature,scope:${scope}`, style: "css", skipTsConfig: false, skipFormat: true, unitTestRunner: "jest", linter: "eslint", }); // Add custom files const projectConfig = readProjectConfiguration( tree, `${scope}-feature-${name}`, ); const projectNames = names(name); generateFiles( tree, joinPathFragments(__dirname, "files"), projectConfig.sourceRoot, { ...projectNames, scope, tmpl: "", }, ); await formatFiles(tree); } ``` ### Template 5: CI Configuration with Affected ```yaml # .github/workflows/ci.yml name: CI on: push: branches: [main] pull_request: branches: [main] env: NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} jobs: main: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-node@v4 with: node-version: 20 cache: "npm" - name: Install dependencies run: npm ci - name: Derive SHAs for affected commands uses: nrwl/nx-set-shas@v4 - name: Run affected lint run: npx nx affected -t lint --parallel=3 - name: Run affected test run: npx nx affected -t test --parallel=3 --configuration=ci - name: Run affected build run: npx nx affected -t build --parallel=3 - name: Run affected e2e run: npx nx affected -t e2e --parallel=1 ``` ### Template 6: Remote Caching Setup ```typescript // nx.json with Nx Cloud { "tasksRunnerOptions": { "default": { "runner": "nx-cloud", "options": { "cacheableOperations": ["build", "lint", "test", "e2e"], "accessToken": "your-nx-cloud-token", "parallel": 3, "cacheDirectory": ".nx/cache" } } }, "nxCloudAccessToken": "your-nx-cloud-token" } // Self-hosted cache with S3 { "tasksRunnerOptions": { "default": { "runner": "@nx-aws-cache/nx-aws-cache", "options": { "cacheableOperations": ["build", "lint", "test"], "awsRegion": "us-east-1", "awsBucket": "my-nx-cache-bucket", "awsProfile": "default" } } } } ``` ## Common Commands ```bash # Generate new library nx g @nx/react:lib feature-auth --directory=libs/web --tags=type:feature,scope:web # Run affected tests nx affected -t test --base=main # View dependency graph nx graph # Run specific project nx build web --configuration=production # Reset cache nx reset # Run migrations nx migrate latest nx migrate --run-migrations ``` ## Best Practices ### Do's - **Use tags consistently** - Enforce with module boundaries - **Enable caching early** - Significant CI savings - **Keep libs focused** - Single responsibility - **Use generators** - Ensure consistency - **Document boundaries** - Help new developers ### Don'ts - **Don't create circular deps** - Graph should be acyclic - **Don't skip affected** - Test only what changed - **Don't ignore boundaries** - Tech debt accumulates - **Don't over-granularize** - Balance lib count ## Resources - [Nx Documentation](https://nx.dev/getting-started/intro) - [Module Boundaries](https://nx.dev/core-features/enforce-module-boundaries) - [Nx Cloud](https://nx.app/)
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

sql-optimization-patterns

Master SQL query optimization, indexing strategies, and EXPLAIN

coding
⭐1
# SQL Optimization Patterns Transform slow database queries into lightning-fast operations through systematic optimization, proper indexing, and query plan analysis. ## When to Use This Skill - Debugging slow-running queries - Designing performant database schemas - Optimizing application response times - Reducing database load and costs - Improving scalability for growing datasets - Analyzing EXPLAIN query plans - Implementing efficient indexes - Resolving N+1 query problems ## Core Concepts ### 1. Query Execution Plans (EXPLAIN) Understanding EXPLAIN output is fundamental to optimization. **PostgreSQL EXPLAIN:** ```sql -- Basic explain EXPLAIN SELECT * FROM users WHERE email = 'user@example.com'; -- With actual execution stats EXPLAIN ANALYZE SELECT * FROM users WHERE email = 'user@example.com'; -- Verbose output with more details EXPLAIN (ANALYZE, BUFFERS, VERBOSE) SELECT u.*, o.order_total FROM users u JOIN orders o ON u.id = o.user_id WHERE u.created_at > NOW() - INTERVAL '30 days'; ``` **Key Metrics to Watch:** - **Seq Scan**: Full table scan (usually slow for large tables) - **Index Scan**: Using index (good) - **Index Only Scan**: Using index without touching table (best) - **Nested Loop**: Join method (okay for small datasets) - **Hash Join**: Join method (good for larger datasets) - **Merge Join**: Join method (good for sorted data) - **Cost**: Estimated query cost (lower is better) - **Rows**: Estimated rows returned - **Actual Time**: Real execution time ### 2. Index Strategies Indexes are the most powerful optimization tool. **Index Types:** - **B-Tree**: Default, good for equality and range queries - **Hash**: Only for equality (=) comparisons - **GIN**: Full-text search, array queries, JSONB - **GiST**: Geometric data, full-text search - **BRIN**: Block Range INdex for very large tables with correlation ```sql -- Standard B-Tree index CREATE INDEX idx_users_email ON users(email); -- Composite index (order matters!) CREATE INDEX idx_orders_user_status ON orders(user_id, status); -- Partial index (index subset of rows) CREATE INDEX idx_active_users ON users(email) WHERE status = 'active'; -- Expression index CREATE INDEX idx_users_lower_email ON users(LOWER(email)); -- Covering index (include additional columns) CREATE INDEX idx_users_email_covering ON users(email) INCLUDE (name, created_at); -- Full-text search index CREATE INDEX idx_posts_search ON posts USING GIN(to_tsvector('english', title || ' ' || body)); -- JSONB index CREATE INDEX idx_metadata ON events USING GIN(metadata); ``` ### 3. Query Optimization Patterns **Avoid SELECT \*:** ```sql -- Bad: Fetches unnecessary columns SELECT * FROM users WHERE id = 123; -- Good: Fetch only what you need SELECT id, email, name FROM users WHERE id = 123; ``` **Use WHERE Clause Efficiently:** ```sql -- Bad: Function prevents index usage SELECT * FROM users WHERE LOWER(email) = 'user@example.com'; -- Good: Create functional index or use exact match CREATE INDEX idx_users_email_lower ON users(LOWER(email)); -- Then: SELECT * FROM users WHERE LOWER(email) = 'user@example.com'; -- Or store normalized data SELECT * FROM users WHERE email = 'user@example.com'; ``` **Optimize JOINs:** ```sql -- Bad: Cartesian product then filter SELECT u.name, o.total FROM users u, orders o WHERE u.id = o.user_id AND u.created_at > '2024-01-01'; -- Good: Filter before join SELECT u.name, o.total FROM users u JOIN orders o ON u.id = o.user_id WHERE u.created_at > '2024-01-01'; -- Better: Filter both tables SELECT u.name, o.total FROM (SELECT * FROM users WHERE created_at > '2024-01-01') u JOIN orders o ON u.id = o.user_id; ``` ## Optimization Patterns ### Pattern 1: Eliminate N+1 Queries **Problem: N+1 Query Anti-Pattern** ```python # Bad: Executes N+1 queries users = db.query("SELECT * FROM users LIMIT 10") for user in users: orders = db.query("SELECT * FROM orders WHERE user_id = ?", user.id) # Process orders ``` **Solution: Use JOINs or Batch Loading** ```sql -- Solution 1: JOIN SELECT u.id, u.name, o.id as order_id, o.total FROM users u LEFT JOIN orders o ON u.id = o.user_id WHERE u.id IN (1, 2, 3, 4, 5); -- Solution 2: Batch query SELECT * FROM orders WHERE user_id IN (1, 2, 3, 4, 5); ``` ```python # Good: Single query with JOIN or batch load # Using JOIN results = db.query(""" SELECT u.id, u.name, o.id as order_id, o.total FROM users u LEFT JOIN orders o ON u.id = o.user_id WHERE u.id IN (1, 2, 3, 4, 5) """) # Or batch load users = db.query("SELECT * FROM users LIMIT 10") user_ids = [u.id for u in users] orders = db.query( "SELECT * FROM orders WHERE user_id IN (?)", user_ids ) # Group orders by user_id orders_by_user = {} for order in orders: orders_by_user.setdefault(order.user_id, []).append(order) ``` ### Pattern 2: Optimize Pagination **Bad: OFFSET on Large Tables** ```sql -- Slow for large offsets SELECT * FROM users ORDER BY created_at DESC LIMIT 20 OFFSET 100000; -- Very slow! ``` **Good: Cursor-Based Pagination** ```sql -- Much faster: Use cursor (last seen ID) SELECT * FROM users WHERE created_at < '2024-01-15 10:30:00' -- Last cursor ORDER BY created_at DESC LIMIT 20; -- With composite sorting SELECT * FROM users WHERE (created_at, id) < ('2024-01-15 10:30:00', 12345) ORDER BY created_at DESC, id DESC LIMIT 20; -- Requires index CREATE INDEX idx_users_cursor ON users(created_at DESC, id DESC); ``` ### Pattern 3: Aggregate Efficiently **Optimize COUNT Queries:** ```sql -- Bad: Counts all rows SELECT COUNT(*) FROM orders; -- Slow on large tables -- Good: Use estimates for approximate counts SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'orders'; -- Good: Filter before counting SELECT COUNT(*) FROM orders WHERE created_at > NOW() - INTERVAL '7 days'; -- Better: Use index-only scan CREATE INDEX idx_orders_created ON orders(created_at); SELECT COUNT(*) FROM orders WHERE created_at > NOW() - INTERVAL '7 days'; ``` **Optimize GROUP BY:** ```sql -- Bad: Group by then filter SELECT user_id, COUNT(*) as order_count FROM orders GROUP BY user_id HAVING COUNT(*) > 10; -- Better: Filter first, then group (if possible) SELECT user_id, COUNT(*) as order_count FROM orders WHERE status = 'completed' GROUP BY user_id HAVING COUNT(*) > 10; -- Best: Use covering index CREATE INDEX idx_orders_user_status ON orders(user_id, status); ``` ### Pattern 4: Subquery Optimization **Transform Correlated Subqueries:** ```sql -- Bad: Correlated subquery (runs for each row) SELECT u.name, u.email, (SELECT COUNT(*) FROM orders o WHERE o.user_id = u.id) as order_count FROM users u; -- Good: JOIN with aggregation SELECT u.name, u.email, COUNT(o.id) as order_count FROM users u LEFT JOIN orders o ON o.user_id = u.id GROUP BY u.id, u.name, u.email; -- Better: Use window functions SELECT DISTINCT ON (u.id) u.name, u.email, COUNT(o.id) OVER (PARTITION BY u.id) as order_count FROM users u LEFT JOIN orders o ON o.user_id = u.id; ``` **Use CTEs for Clarity:** ```sql -- Using Common Table Expressions WITH recent_users AS ( SELECT id, name, email FROM users WHERE created_at > NOW() - INTERVAL '30 days' ), user_order_counts AS ( SELECT user_id, COUNT(*) as order_count FROM orders WHERE created_at > NOW() - INTERVAL '30 days' GROUP BY user_id ) SELECT ru.name, ru.email, COALESCE(uoc.order_count, 0) as orders FROM recent_users ru LEFT JOIN user_order_counts uoc ON ru.id = uoc.user_id; ``` ### Pattern 5: Batch Operations **Batch INSERT:** ```sql -- Bad: Multiple individual inserts INSERT INTO users (name, email) VALUES ('Alice', 'alice@example.com'); INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); INSERT INTO users (name, email) VALUES ('Carol', 'carol@example.com'); -- Good: Batch insert INSERT INTO users (name, email) VALUES ('Alice', 'alice@example.com'), ('Bob', 'bob@example.com'), ('Carol', 'carol@example.com'); -- Better: Use COPY for bulk inserts (PostgreSQL) COPY users (name, email) FROM '/tmp/users.csv' CSV HEADER; ``` **Batch UPDATE:** ```sql -- Bad: Update in loop UPDATE users SET status = 'active' WHERE id = 1; UPDATE users SET status = 'active' WHERE id = 2; -- ... repeat for many IDs -- Good: Single UPDATE with IN clause UPDATE users SET status = 'active' WHERE id IN (1, 2, 3, 4, 5, ...); -- Better: Use temporary table for large batches CREATE TEMP TABLE temp_user_updates (id INT, new_status VARCHAR); INSERT INTO temp_user_updates VALUES (1, 'active'), (2, 'active'), ...; UPDATE users u SET status = t.new_status FROM temp_user_updates t WHERE u.id = t.id; ``` ## Advanced Techniques ### Materialized Views Pre-compute expensive queries. ```sql -- Create materialized view CREATE MATERIALIZED VIEW user_order_summary AS SELECT u.id, u.name, COUNT(o.id) as total_orders, SUM(o.total) as total_spent, MAX(o.created_at) as last_order_date FROM users u LEFT JOIN orders o ON u.id = o.user_id GROUP BY u.id, u.name; -- Add index to materialized view CREATE INDEX idx_user_summary_spent ON user_order_summary(total_spent DESC); -- Refresh materialized view REFRESH MATERIALIZED VIEW user_order_summary; -- Concurrent refresh (PostgreSQL) REFRESH MATERIALIZED VIEW CONCURRENTLY user_order_summary; -- Query materialized view (very fast) SELECT * FROM user_order_summary WHERE total_spent > 1000 ORDER BY total_spent DESC; ``` ### Partitioning Split large tables for better performance. ```sql -- Range partitioning by date (PostgreSQL) CREATE TABLE orders ( id SERIAL, user_id INT, total DECIMAL, created_at TIMESTAMP ) PARTITION BY RANGE (created_at); -- Create partitions CREATE TABLE orders_2024_q1 PARTITION OF orders FOR VALUES FROM ('2024-01-01') TO ('2024-04-01'); CREATE TABLE orders_2024_q2 PARTITION OF orders FOR VALUES FROM ('2024-04-01') TO ('2024-07-01'); -- Queries automatically use appropriate partition SELECT * FROM orders WHERE created_at BETWEEN '2024-02-01' AND '2024-02-28'; -- Only scans orders_2024_q1 partition ``` ### Query Hints and Optimization ```sql -- Force index usage (MySQL) SELECT * FROM users USE INDEX (idx_users_email) WHERE email = 'user@example.com'; -- Parallel query (PostgreSQL) SET max_parallel_workers_per_gather = 4; SELECT * FROM large_table WHERE condition; -- Join hints (PostgreSQL) SET enable_nestloop = OFF; -- Force hash or merge join ``` ## Best Practices 1. **Index Selectively**: Too many indexes slow down writes 2. **Monitor Query Performance**: Use slow query logs 3. **Keep Statistics Updated**: Run ANALYZE regularly 4. **Use Appropriate Data Types**: Smaller types = better performance 5. **Normalize Thoughtfully**: Balance normalization vs performance 6. **Cache Frequently Accessed Data**: Use application-level caching 7. **Connection Pooling**: Reuse database connections 8. **Regular Maintenance**: VACUUM, ANALYZE, rebuild indexes ```sql -- Update statistics ANALYZE users; ANALYZE VERBOSE orders; -- Vacuum (PostgreSQL) VACUUM ANALYZE users; VACUUM FULL users; -- Reclaim space (locks table) -- Reindex REINDEX INDEX idx_users_email; REINDEX TABLE users; ``` ## Common Pitfalls - **Over-Indexing**: Each index slows down INSERT/UPDATE/DELETE - **Unused Indexes**: Waste space and slow writes - **Missing Indexes**: Slow queries, full table scans - **Implicit Type Conversion**: Prevents index usage - **OR Conditions**: Can't use indexes efficiently - **LIKE with Leading Wildcard**: `LIKE '%abc'` can't use index - **Function in WHERE**: Prevents index usage unless functional index exists ## Monitoring Queries ```sql -- Find slow queries (PostgreSQL) SELECT query, calls, total_time, mean_time FROM pg_stat_statements ORDER BY mean_time DESC LIMIT 10; -- Find missing indexes (PostgreSQL) SELECT schemaname, tablename, seq_scan, seq_tup_read, idx_scan, seq_tup_read / seq_scan AS avg_seq_tup_read FROM pg_stat_user_tables WHERE seq_scan > 0 ORDER BY seq_tup_read DESC LIMIT 10; -- Find unused indexes (PostgreSQL) SELECT schemaname, tablename, indexname, idx_scan, idx_tup_read, idx_tup_fetch FROM pg_stat_user_indexes WHERE idx_scan = 0 ORDER BY pg_relation_size(indexrelid) DESC; ``` ## Resources - **references/postgres-optimization-guide.md**: PostgreSQL-specific optimization - **references/mysql-optimization-guide.md**: MySQL/MariaDB optimization - **references/query-plan-analysis.md**: Deep dive into EXPLAIN plans - **assets/index-strategy-checklist.md**: When and how to create indexes - **assets/query-optimization-checklist.md**: Step-by-step optimization guide - **scripts/analyze-slow-queries.sql**: Identify slow queries in your database - **scripts/index-recommendations.sql**: Generate index recommendations
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

dependency-upgrade

Manage major dependency version upgrades with compatibility

coding
⭐1
# Dependency Upgrade Master major dependency version upgrades, compatibility analysis, staged upgrade strategies, and comprehensive testing approaches. ## When to Use This Skill - Upgrading major framework versions - Updating security-vulnerable dependencies - Modernizing legacy dependencies - Resolving dependency conflicts - Planning incremental upgrade paths - Testing compatibility matrices - Automating dependency updates ## Semantic Versioning Review ``` MAJOR.MINOR.PATCH (e.g., 2.3.1) MAJOR: Breaking changes MINOR: New features, backward compatible PATCH: Bug fixes, backward compatible ^2.3.1 = >=2.3.1 <3.0.0 (minor updates) ~2.3.1 = >=2.3.1 <2.4.0 (patch updates) 2.3.1 = exact version ``` ## Dependency Analysis ### Audit Dependencies ```bash # npm npm outdated npm audit npm audit fix # yarn yarn outdated yarn audit # Check for major updates npx npm-check-updates npx npm-check-updates -u # Update package.json ``` ### Analyze Dependency Tree ```bash # See why a package is installed npm ls package-name yarn why package-name # Find duplicate packages npm dedupe yarn dedupe # Visualize dependencies npx madge --image graph.png src/ ``` ## Compatibility Matrix ```javascript // compatibility-matrix.js const compatibilityMatrix = { react: { "16.x": { "react-dom": "^16.0.0", "react-router-dom": "^5.0.0", "@testing-library/react": "^11.0.0", }, "17.x": { "react-dom": "^17.0.0", "react-router-dom": "^5.0.0 || ^6.0.0", "@testing-library/react": "^12.0.0", }, "18.x": { "react-dom": "^18.0.0", "react-router-dom": "^6.0.0", "@testing-library/react": "^13.0.0", }, }, }; function checkCompatibility(packages) { // Validate package versions against matrix } ``` ## Staged Upgrade Strategy ### Phase 1: Planning ```bash # 1. Identify current versions npm list --depth=0 # 2. Check for breaking changes # Read CHANGELOG.md and MIGRATION.md # 3. Create upgrade plan echo "Upgrade order: 1. TypeScript 2. React 3. React Router 4. Testing libraries 5. Build tools" > UPGRADE_PLAN.md ``` ### Phase 2: Incremental Updates ```bash # Don't upgrade everything at once! # Step 1: Update TypeScript npm install typescript@latest # Test npm run test npm run build # Step 2: Update React (one major version at a time) npm install react@17 react-dom@17 # Test again npm run test # Step 3: Continue with other packages npm install react-router-dom@6 # And so on... ``` ### Phase 3: Validation ```javascript // tests/compatibility.test.js describe("Dependency Compatibility", () => { it("should have compatible React versions", () => { const reactVersion = require("react/package.json").version; const reactDomVersion = require("react-dom/package.json").version; expect(reactVersion).toBe(reactDomVersion); }); it("should not have peer dependency warnings", () => { // Run npm ls and check for warnings }); }); ``` ## Breaking Change Handling ### Identifying Breaking Changes ```bash # Check the changelog directly curl https://raw.githubusercontent.com/facebook/react/master/CHANGELOG.md ``` ### Codemod for Automated Fixes ```bash # Run jscodeshift with transform URL npx jscodeshift -t <transform-url> <path> # Example: Rename unsafe lifecycle methods npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/rename-unsafe-lifecycles.js src/ # For TypeScript files npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/rename-unsafe-lifecycles.js --parser=tsx src/ # Dry run to preview changes npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/rename-unsafe-lifecycles.js --dry src/ ``` ### Custom Migration Script ```javascript // migration-script.js const fs = require("fs"); const glob = require("glob"); glob("src/**/*.tsx", (err, files) => { files.forEach((file) => { let content = fs.readFileSync(file, "utf8"); // Replace old API with new API content = content.replace( /componentWillMount/g, "UNSAFE_componentWillMount", ); // Update imports content = content.replace( /import { Component } from 'react'/g, "import React, { Component } from 'react'", ); fs.writeFileSync(file, content); }); }); ``` ## Testing Strategy ### Unit Tests ```javascript // Ensure tests pass before and after upgrade npm run test // Update test utilities if needed npm install @testing-library/react@latest ``` ### Integration Tests ```javascript // tests/integration/app.test.js describe("App Integration", () => { it("should render without crashing", () => { render(<App />); }); it("should handle navigation", () => { const { getByText } = render(<App />); fireEvent.click(getByText("Navigate")); expect(screen.getByText("New Page")).toBeInTheDocument(); }); }); ``` ### Visual Regression Tests ```javascript // visual-regression.test.js describe("Visual Regression", () => { it("should match snapshot", () => { const { container } = render(<App />); expect(container.firstChild).toMatchSnapshot(); }); }); ``` ### E2E Tests ```javascript // cypress/e2e/app.cy.js describe("E2E Tests", () => { it("should complete user flow", () => { cy.visit("/"); cy.get('[data-testid="login"]').click(); cy.get('input[name="email"]').type("user@example.com"); cy.get('button[type="submit"]').click(); cy.url().should("include", "/dashboard"); }); }); ``` ## Automated Dependency Updates ### Renovate Configuration ```json // renovate.json { "extends": ["config:base"], "packageRules": [ { "matchUpdateTypes": ["minor", "patch"], "automerge": true }, { "matchUpdateTypes": ["major"], "automerge": false, "labels": ["major-update"] } ], "schedule": ["before 3am on Monday"], "timezone": "America/New_York" } ``` ### Dependabot Configuration ```yaml # .github/dependabot.yml version: 2 updates: - package-ecosystem: "npm" directory: "/" schedule: interval: "weekly" open-pull-requests-limit: 5 reviewers: - "team-leads" commit-message: prefix: "chore" include: "scope" ``` ## Rollback Plan ```javascript // rollback.sh #!/bin/bash # Save current state git stash git checkout -b upgrade-branch # Attempt upgrade npm install package@latest # Run tests if npm run test; then echo "Upgrade successful" git add package.json package-lock.json git commit -m "chore: upgrade package" else echo "Upgrade failed, rolling back" git checkout main git branch -D upgrade-branch npm install # Restore from package-lock.json fi ``` ## Common Upgrade Patterns ### Lock File Management ```bash # npm npm install --package-lock-only # Update lock file only npm ci # Clean install from lock file # yarn yarn install --frozen-lockfile # CI mode yarn upgrade-interactive # Interactive upgrades ``` ### Peer Dependency Resolution ```bash # npm 7+: strict peer dependencies npm install --legacy-peer-deps # Ignore peer deps # npm 8+: override peer dependencies npm install --force ``` ### Workspace Upgrades ```bash # Update all workspace packages npm install --workspaces # Update specific workspace npm install package@latest --workspace=packages/app ``` ## Resources - **references/semver.md**: Semantic versioning guide - **references/compatibility-matrix.md**: Common compatibility issues - **references/staged-upgrades.md**: Incremental upgrade strategies - **references/testing-strategy.md**: Comprehensive testing approaches - **assets/upgrade-checklist.md**: Step-by-step checklist - **assets/compatibility-matrix.csv**: Version compatibility table - **scripts/audit-dependencies.sh**: Dependency audit script ## Best Practices 1. **Read Changelogs**: Understand what changed 2. **Upgrade Incrementally**: One major version at a time 3. **Test Thoroughly**: Unit, integration, E2E tests 4. **Check Peer Dependencies**: Resolve conflicts early 5. **Use Lock Files**: Ensure reproducible installs 6. **Automate Updates**: Use Renovate or Dependabot 7. **Monitor**: Watch for runtime errors post-upgrade 8. **Document**: Keep upgrade notes ## Upgrade Checklist ```markdown Pre-Upgrade: - [ ] Review current dependency versions - [ ] Read changelogs for breaking changes - [ ] Create feature branch - [ ] Backup current state (git tag) - [ ] Run full test suite (baseline) During Upgrade: - [ ] Upgrade one dependency at a time - [ ] Update peer dependencies - [ ] Fix TypeScript errors - [ ] Update tests if needed - [ ] Run test suite after each upgrade - [ ] Check bundle size impact Post-Upgrade: - [ ] Full regression testing - [ ] Performance testing - [ ] Update documentation - [ ] Deploy to staging - [ ] Monitor for errors - [ ] Deploy to production ``` ## Common Pitfalls - Upgrading all dependencies at once - Not testing after each upgrade - Ignoring peer dependency warnings - Forgetting to update lock file - Not reading breaking change notes - Skipping major versions - Not having rollback plan
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

nextjs-app-router-patterns

Master Next.js 14+ App Router with Server Components, streaming,

coding
⭐1
# Next.js App Router Patterns Comprehensive patterns for Next.js 14+ App Router architecture, Server Components, and modern full-stack React development. ## When to Use This Skill - Building new Next.js applications with App Router - Migrating from Pages Router to App Router - Implementing Server Components and streaming - Setting up parallel and intercepting routes - Optimizing data fetching and caching - Building full-stack features with Server Actions ## Core Concepts ### 1. Rendering Modes | Mode | Where | When to Use | | --------------------- | ------------ | ----------------------------------------- | | **Server Components** | Server only | Data fetching, heavy computation, secrets | | **Client Components** | Browser | Interactivity, hooks, browser APIs | | **Static** | Build time | Content that rarely changes | | **Dynamic** | Request time | Personalized or real-time data | | **Streaming** | Progressive | Large pages, slow data sources | ### 2. File Conventions ``` app/ ├── layout.tsx # Shared UI wrapper ├── page.tsx # Route UI ├── loading.tsx # Loading UI (Suspense) ├── error.tsx # Error boundary ├── not-found.tsx # 404 UI ├── route.ts # API endpoint ├── template.tsx # Re-mounted layout ├── default.tsx # Parallel route fallback └── opengraph-image.tsx # OG image generation ``` ## Quick Start ```typescript // app/layout.tsx import { Inter } from 'next/font/google' import { Providers } from './providers' const inter = Inter({ subsets: ['latin'] }) export const metadata = { title: { default: 'My App', template: '%s | My App' }, description: 'Built with Next.js App Router', } export default function RootLayout({ children, }: { children: React.ReactNode }) { return ( <html lang="en" suppressHydrationWarning> <body className={inter.className}> <Providers>{children}</Providers> </body> </html> ) } // app/page.tsx - Server Component by default async function getProducts() { const res = await fetch('https://api.example.com/products', { next: { revalidate: 3600 }, // ISR: revalidate every hour }) return res.json() } export default async function HomePage() { const products = await getProducts() return ( <main> <h1>Products</h1> <ProductGrid products={products} /> </main> ) } ``` ## Patterns ### Pattern 1: Server Components with Data Fetching ```typescript // app/products/page.tsx import { Suspense } from 'react' import { ProductList, ProductListSkeleton } from '@/components/products' import { FilterSidebar } from '@/components/filters' interface SearchParams { category?: string sort?: 'price' | 'name' | 'date' page?: string } export default async function ProductsPage({ searchParams, }: { searchParams: Promise<SearchParams> }) { const params = await searchParams return ( <div className="flex gap-8"> <FilterSidebar /> <Suspense key={JSON.stringify(params)} fallback={<ProductListSkeleton />} > <ProductList category={params.category} sort={params.sort} page={Number(params.page) || 1} /> </Suspense> </div> ) } // components/products/ProductList.tsx - Server Component async function getProducts(filters: ProductFilters) { const res = await fetch( `${process.env.API_URL}/products?${new URLSearchParams(filters)}`, { next: { tags: ['products'] } } ) if (!res.ok) throw new Error('Failed to fetch products') return res.json() } export async function ProductList({ category, sort, page }: ProductFilters) { const { products, totalPages } = await getProducts({ category, sort, page }) return ( <div> <div className="grid grid-cols-3 gap-4"> {products.map((product) => ( <ProductCard key={product.id} product={product} /> ))} </div> <Pagination currentPage={page} totalPages={totalPages} /> </div> ) } ``` ### Pattern 2: Client Components with 'use client' ```typescript // components/products/AddToCartButton.tsx 'use client' import { useState, useTransition } from 'react' import { addToCart } from '@/app/actions/cart' export function AddToCartButton({ productId }: { productId: string }) { const [isPending, startTransition] = useTransition() const [error, setError] = useState<string | null>(null) const handleClick = () => { setError(null) startTransition(async () => { const result = await addToCart(productId) if (result.error) { setError(result.error) } }) } return ( <div> <button onClick={handleClick} disabled={isPending} className="btn-primary" > {isPending ? 'Adding...' : 'Add to Cart'} </button> {error && <p className="text-red-500 text-sm">{error}</p>} </div> ) } ``` ### Pattern 3: Server Actions ```typescript // app/actions/cart.ts "use server"; import { revalidateTag } from "next/cache"; import { cookies } from "next/headers"; import { redirect } from "next/navigation"; export async function addToCart(productId: string) { const cookieStore = await cookies(); const sessionId = cookieStore.get("session")?.value; if (!sessionId) { redirect("/login"); } try { await db.cart.upsert({ where: { sessionId_productId: { sessionId, productId } }, update: { quantity: { increment: 1 } }, create: { sessionId, productId, quantity: 1 }, }); revalidateTag("cart"); return { success: true }; } catch (error) { return { error: "Failed to add item to cart" }; } } export async function checkout(formData: FormData) { const address = formData.get("address") as string; const payment = formData.get("payment") as string; // Validate if (!address || !payment) { return { error: "Missing required fields" }; } // Process order const order = await processOrder({ address, payment }); // Redirect to confirmation redirect(`/orders/${order.id}/confirmation`); } ``` ### Pattern 4: Parallel Routes ```typescript // app/dashboard/layout.tsx export default function DashboardLayout({ children, analytics, team, }: { children: React.ReactNode analytics: React.ReactNode team: React.ReactNode }) { return ( <div className="dashboard-grid"> <main>{children}</main> <aside className="analytics-panel">{analytics}</aside> <aside className="team-panel">{team}</aside> </div> ) } // app/dashboard/@analytics/page.tsx export default async function AnalyticsSlot() { const stats = await getAnalytics() return <AnalyticsChart data={stats} /> } // app/dashboard/@analytics/loading.tsx export default function AnalyticsLoading() { return <ChartSkeleton /> } // app/dashboard/@team/page.tsx export default async function TeamSlot() { const members = await getTeamMembers() return <TeamList members={members} /> } ``` ### Pattern 5: Intercepting Routes (Modal Pattern) ```typescript // File structure for photo modal // app/ // ├── @modal/ // │ ├── (.)photos/[id]/page.tsx # Intercept // │ └── default.tsx // ├── photos/ // │ └── [id]/page.tsx # Full page // └── layout.tsx // app/@modal/(.)photos/[id]/page.tsx import { Modal } from '@/components/Modal' import { PhotoDetail } from '@/components/PhotoDetail' export default async function PhotoModal({ params, }: { params: Promise<{ id: string }> }) { const { id } = await params const photo = await getPhoto(id) return ( <Modal> <PhotoDetail photo={photo} /> </Modal> ) } // app/photos/[id]/page.tsx - Full page version export default async function PhotoPage({ params, }: { params: Promise<{ id: string }> }) { const { id } = await params const photo = await getPhoto(id) return ( <div className="photo-page"> <PhotoDetail photo={photo} /> <RelatedPhotos photoId={id} /> </div> ) } // app/layout.tsx export default function RootLayout({ children, modal, }: { children: React.ReactNode modal: React.ReactNode }) { return ( <html> <body> {children} {modal} </body> </html> ) } ``` ### Pattern 6: Streaming with Suspense ```typescript // app/product/[id]/page.tsx import { Suspense } from 'react' export default async function ProductPage({ params, }: { params: Promise<{ id: string }> }) { const { id } = await params // This data loads first (blocking) const product = await getProduct(id) return ( <div> {/* Immediate render */} <ProductHeader product={product} /> {/* Stream in reviews */} <Suspense fallback={<ReviewsSkeleton />}> <Reviews productId={id} /> </Suspense> {/* Stream in recommendations */} <Suspense fallback={<RecommendationsSkeleton />}> <Recommendations productId={id} /> </Suspense> </div> ) } // These components fetch their own data async function Reviews({ productId }: { productId: string }) { const reviews = await getReviews(productId) // Slow API return <ReviewList reviews={reviews} /> } async function Recommendations({ productId }: { productId: string }) { const products = await getRecommendations(productId) // ML-based, slow return <ProductCarousel products={products} /> } ``` ### Pattern 7: Route Handlers (API Routes) ```typescript // app/api/products/route.ts import { NextRequest, NextResponse } from "next/server"; export async function GET(request: NextRequest) { const searchParams = request.nextUrl.searchParams; const category = searchParams.get("category"); const products = await db.product.findMany({ where: category ? { category } : undefined, take: 20, }); return NextResponse.json(products); } export async function POST(request: NextRequest) { const body = await request.json(); const product = await db.product.create({ data: body, }); return NextResponse.json(product, { status: 201 }); } // app/api/products/[id]/route.ts export async function GET( request: NextRequest, { params }: { params: Promise<{ id: string }> }, ) { const { id } = await params; const product = await db.product.findUnique({ where: { id } }); if (!product) { return NextResponse.json({ error: "Product not found" }, { status: 404 }); } return NextResponse.json(product); } ``` ### Pattern 8: Metadata and SEO ```typescript // app/products/[slug]/page.tsx import { Metadata } from 'next' import { notFound } from 'next/navigation' type Props = { params: Promise<{ slug: string }> } export async function generateMetadata({ params }: Props): Promise<Metadata> { const { slug } = await params const product = await getProduct(slug) if (!product) return {} return { title: product.name, description: product.description, openGraph: { title: product.name, description: product.description, images: [{ url: product.image, width: 1200, height: 630 }], }, twitter: { card: 'summary_large_image', title: product.name, description: product.description, images: [product.image], }, } } export async function generateStaticParams() { const products = await db.product.findMany({ select: { slug: true } }) return products.map((p) => ({ slug: p.slug })) } export default async function ProductPage({ params }: Props) { const { slug } = await params const product = await getProduct(slug) if (!product) notFound() return <ProductDetail product={product} /> } ``` ## Caching Strategies ### Data Cache ```typescript // No cache (always fresh) fetch(url, { cache: "no-store" }); // Cache forever (static) fetch(url, { cache: "force-cache" }); // ISR - revalidate after 60 seconds fetch(url, { next: { revalidate: 60 } }); // Tag-based invalidation fetch(url, { next: { tags: ["products"] } }); // Invalidate via Server Action ("use server"); import { revalidateTag, revalidatePath } from "next/cache"; export async function updateProduct(id: string, data: ProductData) { await db.product.update({ where: { id }, data }); revalidateTag("products"); revalidatePath("/products"); } ``` ## Best Practices ### Do's - **Start with Server Components** - Add 'use client' only when needed - **Colocate data fetching** - Fetch data where it's used - **Use Suspense boundaries** - Enable streaming for slow data - **Leverage parallel routes** - Independent loading states - **Use Server Actions** - For mutations with progressive enhancement ### Don'ts - **Don't pass serializable data** - Server → Client boundary limitations - **Don't use hooks in Server Components** - No useState, useEffect - **Don't fetch in Client Components** - Use Server Components or React Query - **Don't over-nest layouts** - Each layout adds to the component tree - **Don't ignore loading states** - Always provide loading.tsx or Suspense ## Resources - [Next.js App Router Documentation](https://nextjs.org/docs/app) - [Server Components RFC](https://github.com/reactjs/rfcs/blob/main/text/0188-server-components.md) - [Vercel Templates](https://vercel.com/templates/next.js)
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

godot-gdscript-patterns

Master Godot 4 GDScript patterns including signals, scenes, state

coding
⭐1
# Godot GDScript Patterns Production patterns for Godot 4.x game development with GDScript, covering architecture, signals, scenes, and optimization. ## When to Use This Skill - Building games with Godot 4 - Implementing game systems in GDScript - Designing scene architecture - Managing game state - Optimizing GDScript performance - Learning Godot best practices ## Core Concepts ### 1. Godot Architecture ``` Node: Base building block ├── Scene: Reusable node tree (saved as .tscn) ├── Resource: Data container (saved as .tres) ├── Signal: Event communication └── Group: Node categorization ``` ### 2. GDScript Basics ```gdscript class_name Player extends CharacterBody2D # Signals signal health_changed(new_health: int) signal died # Exports (Inspector-editable) @export var speed: float = 200.0 @export var max_health: int = 100 @export_range(0, 1) var damage_reduction: float = 0.0 @export_group("Combat") @export var attack_damage: int = 10 @export var attack_cooldown: float = 0.5 # Onready (initialized when ready) @onready var sprite: Sprite2D = $Sprite2D @onready var animation: AnimationPlayer = $AnimationPlayer @onready var hitbox: Area2D = $Hitbox # Private variables (convention: underscore prefix) var _health: int var _can_attack: bool = true func _ready() -> void: _health = max_health func _physics_process(delta: float) -> void: var direction := Input.get_vector("left", "right", "up", "down") velocity = direction * speed move_and_slide() func take_damage(amount: int) -> void: var actual_damage := int(amount * (1.0 - damage_reduction)) _health = max(_health - actual_damage, 0) health_changed.emit(_health) if _health <= 0: died.emit() ``` ## Patterns ### Pattern 1: State Machine ```gdscript # state_machine.gd class_name StateMachine extends Node signal state_changed(from_state: StringName, to_state: StringName) @export var initial_state: State var current_state: State var states: Dictionary = {} func _ready() -> void: # Register all State children for child in get_children(): if child is State: states[child.name] = child child.state_machine = self child.process_mode = Node.PROCESS_MODE_DISABLED # Start initial state if initial_state: current_state = initial_state current_state.process_mode = Node.PROCESS_MODE_INHERIT current_state.enter() func _process(delta: float) -> void: if current_state: current_state.update(delta) func _physics_process(delta: float) -> void: if current_state: current_state.physics_update(delta) func _unhandled_input(event: InputEvent) -> void: if current_state: current_state.handle_input(event) func transition_to(state_name: StringName, msg: Dictionary = {}) -> void: if not states.has(state_name): push_error("State '%s' not found" % state_name) return var previous_state := current_state previous_state.exit() previous_state.process_mode = Node.PROCESS_MODE_DISABLED current_state = states[state_name] current_state.process_mode = Node.PROCESS_MODE_INHERIT current_state.enter(msg) state_changed.emit(previous_state.name, current_state.name) ``` ```gdscript # state.gd class_name State extends Node var state_machine: StateMachine func enter(_msg: Dictionary = {}) -> void: pass func exit() -> void: pass func update(_delta: float) -> void: pass func physics_update(_delta: float) -> void: pass func handle_input(_event: InputEvent) -> void: pass ``` ```gdscript # player_idle.gd class_name PlayerIdle extends State @export var player: Player func enter(_msg: Dictionary = {}) -> void: player.animation.play("idle") func physics_update(_delta: float) -> void: var direction := Input.get_vector("left", "right", "up", "down") if direction != Vector2.ZERO: state_machine.transition_to("Move") func handle_input(event: InputEvent) -> void: if event.is_action_pressed("attack"): state_machine.transition_to("Attack") elif event.is_action_pressed("jump"): state_machine.transition_to("Jump") ``` ### Pattern 2: Autoload Singletons ```gdscript # game_manager.gd (Add to Project Settings > Autoload) extends Node signal game_started signal game_paused(is_paused: bool) signal game_over(won: bool) signal score_changed(new_score: int) enum GameState { MENU, PLAYING, PAUSED, GAME_OVER } var state: GameState = GameState.MENU var score: int = 0: set(value): score = value score_changed.emit(score) var high_score: int = 0 func _ready() -> void: process_mode = Node.PROCESS_MODE_ALWAYS _load_high_score() func _input(event: InputEvent) -> void: if event.is_action_pressed("pause") and state == GameState.PLAYING: toggle_pause() func start_game() -> void: score = 0 state = GameState.PLAYING game_started.emit() func toggle_pause() -> void: var is_paused := state != GameState.PAUSED if is_paused: state = GameState.PAUSED get_tree().paused = true else: state = GameState.PLAYING get_tree().paused = false game_paused.emit(is_paused) func end_game(won: bool) -> void: state = GameState.GAME_OVER if score > high_score: high_score = score _save_high_score() game_over.emit(won) func add_score(points: int) -> void: score += points func _load_high_score() -> void: if FileAccess.file_exists("user://high_score.save"): var file := FileAccess.open("user://high_score.save", FileAccess.READ) high_score = file.get_32() func _save_high_score() -> void: var file := FileAccess.open("user://high_score.save", FileAccess.WRITE) file.store_32(high_score) ``` ```gdscript # event_bus.gd (Global signal bus) extends Node # Player events signal player_spawned(player: Node2D) signal player_died(player: Node2D) signal player_health_changed(health: int, max_health: int) # Enemy events signal enemy_spawned(enemy: Node2D) signal enemy_died(enemy: Node2D, position: Vector2) # Item events signal item_collected(item_type: StringName, value: int) signal powerup_activated(powerup_type: StringName) # Level events signal level_started(level_number: int) signal level_completed(level_number: int, time: float) signal checkpoint_reached(checkpoint_id: int) ``` ### Pattern 3: Resource-based Data ```gdscript # weapon_data.gd class_name WeaponData extends Resource @export var name: StringName @export var damage: int @export var attack_speed: float @export var range: float @export_multiline var description: String @export var icon: Texture2D @export var projectile_scene: PackedScene @export var sound_attack: AudioStream ``` ```gdscript # character_stats.gd class_name CharacterStats extends Resource signal stat_changed(stat_name: StringName, new_value: float) @export var max_health: float = 100.0 @export var attack: float = 10.0 @export var defense: float = 5.0 @export var speed: float = 200.0 # Runtime values (not saved) var _current_health: float func _init() -> void: _current_health = max_health func get_current_health() -> float: return _current_health func take_damage(amount: float) -> float: var actual_damage := maxf(amount - defense, 1.0) _current_health = maxf(_current_health - actual_damage, 0.0) stat_changed.emit("health", _current_health) return actual_damage func heal(amount: float) -> void: _current_health = minf(_current_health + amount, max_health) stat_changed.emit("health", _current_health) func duplicate_for_runtime() -> CharacterStats: var copy := duplicate() as CharacterStats copy._current_health = copy.max_health return copy ``` ```gdscript # Using resources class_name Character extends CharacterBody2D @export var base_stats: CharacterStats @export var weapon: WeaponData var stats: CharacterStats func _ready() -> void: # Create runtime copy to avoid modifying the resource stats = base_stats.duplicate_for_runtime() stats.stat_changed.connect(_on_stat_changed) func attack() -> void: if weapon: print("Attacking with %s for %d damage" % [weapon.name, weapon.damage]) func _on_stat_changed(stat_name: StringName, value: float) -> void: if stat_name == "health" and value <= 0: die() ``` ### Pattern 4: Object Pooling ```gdscript # object_pool.gd class_name ObjectPool extends Node @export var pooled_scene: PackedScene @export var initial_size: int = 10 @export var can_grow: bool = true var _available: Array[Node] = [] var _in_use: Array[Node] = [] func _ready() -> void: _initialize_pool() func _initialize_pool() -> void: for i in initial_size: _create_instance() func _create_instance() -> Node: var instance := pooled_scene.instantiate() instance.process_mode = Node.PROCESS_MODE_DISABLED instance.visible = false add_child(instance) _available.append(instance) # Connect return signal if exists if instance.has_signal("returned_to_pool"): instance.returned_to_pool.connect(_return_to_pool.bind(instance)) return instance func get_instance() -> Node: var instance: Node if _available.is_empty(): if can_grow: instance = _create_instance() _available.erase(instance) else: push_warning("Pool exhausted and cannot grow") return null else: instance = _available.pop_back() instance.process_mode = Node.PROCESS_MODE_INHERIT instance.visible = true _in_use.append(instance) if instance.has_method("on_spawn"): instance.on_spawn() return instance func _return_to_pool(instance: Node) -> void: if not instance in _in_use: return _in_use.erase(instance) if instance.has_method("on_despawn"): instance.on_despawn() instance.process_mode = Node.PROCESS_MODE_DISABLED instance.visible = false _available.append(instance) func return_all() -> void: for instance in _in_use.duplicate(): _return_to_pool(instance) ``` ```gdscript # pooled_bullet.gd class_name PooledBullet extends Area2D signal returned_to_pool @export var speed: float = 500.0 @export var lifetime: float = 5.0 var direction: Vector2 var _timer: float func on_spawn() -> void: _timer = lifetime func on_despawn() -> void: direction = Vector2.ZERO func initialize(pos: Vector2, dir: Vector2) -> void: global_position = pos direction = dir.normalized() rotation = direction.angle() func _physics_process(delta: float) -> void: position += direction * speed * delta _timer -= delta if _timer <= 0: returned_to_pool.emit() func _on_body_entered(body: Node2D) -> void: if body.has_method("take_damage"): body.take_damage(10) returned_to_pool.emit() ``` ### Pattern 5: Component System ```gdscript # health_component.gd class_name HealthComponent extends Node signal health_changed(current: int, maximum: int) signal damaged(amount: int, source: Node) signal healed(amount: int) signal died @export var max_health: int = 100 @export var invincibility_time: float = 0.0 var current_health: int: set(value): var old := current_health current_health = clampi(value, 0, max_health) if current_health != old: health_changed.emit(current_health, max_health) var _invincible: bool = false func _ready() -> void: current_health = max_health func take_damage(amount: int, source: Node = null) -> int: if _invincible or current_health <= 0: return 0 var actual := mini(amount, current_health) current_health -= actual damaged.emit(actual, source) if current_health <= 0: died.emit() elif invincibility_time > 0: _start_invincibility() return actual func heal(amount: int) -> int: var actual := mini(amount, max_health - current_health) current_health += actual if actual > 0: healed.emit(actual) return actual func _start_invincibility() -> void: _invincible = true await get_tree().create_timer(invincibility_time).timeout _invincible = false ``` ```gdscript # hitbox_component.gd class_name HitboxComponent extends Area2D signal hit(hurtbox: HurtboxComponent) @export var damage: int = 10 @export var knockback_force: float = 200.0 var owner_node: Node func _ready() -> void: owner_node = get_parent() area_entered.connect(_on_area_entered) func _on_area_entered(area: Area2D) -> void: if area is HurtboxComponent: var hurtbox := area as HurtboxComponent if hurtbox.owner_node != owner_node: hit.emit(hurtbox) hurtbox.receive_hit(self) ``` ```gdscript # hurtbox_component.gd class_name HurtboxComponent extends Area2D signal hurt(hitbox: HitboxComponent) @export var health_component: HealthComponent var owner_node: Node func _ready() -> void: owner_node = get_parent() func receive_hit(hitbox: HitboxComponent) -> void: hurt.emit(hitbox) if health_component: health_component.take_damage(hitbox.damage, hitbox.owner_node) ``` ### Pattern 6: Scene Management ```gdscript # scene_manager.gd (Autoload) extends Node signal scene_loading_started(scene_path: String) signal scene_loading_progress(progress: float) signal scene_loaded(scene: Node) signal transition_started signal transition_finished @export var transition_scene: PackedScene @export var loading_scene: PackedScene var _current_scene: Node var _transition: CanvasLayer var _loader: ResourceLoader func _ready() -> void: _current_scene = get_tree().current_scene if transition_scene: _transition = transition_scene.instantiate() add_child(_transition) _transition.visible = false func change_scene(scene_path: String, with_transition: bool = true) -> void: if with_transition: await _play_transition_out() _load_scene(scene_path) func change_scene_packed(scene: PackedScene, with_transition: bool = true) -> void: if with_transition: await _play_transition_out() _swap_scene(scene.instantiate()) func _load_scene(path: String) -> void: scene_loading_started.emit(path) # Check if already loaded if ResourceLoader.has_cached(path): var scene := load(path) as PackedScene _swap_scene(scene.instantiate()) return # Async loading ResourceLoader.load_threaded_request(path) while true: var progress := [] var status := ResourceLoader.load_threaded_get_status(path, progress) match status: ResourceLoader.THREAD_LOAD_IN_PROGRESS: scene_loading_progress.emit(progress[0]) await get_tree().process_frame ResourceLoader.THREAD_LOAD_LOADED: var scene := ResourceLoader.load_threaded_get(path) as PackedScene _swap_scene(scene.instantiate()) return _: push_error("Failed to load scene: %s" % path) return func _swap_scene(new_scene: Node) -> void: if _current_scene: _current_scene.queue_free() _current_scene = new_scene get_tree().root.add_child(_current_scene) get_tree().current_scene = _current_scene scene_loaded.emit(_current_scene) await _play_transition_in() func _play_transition_out() -> void: if not _transition: return transition_started.emit() _transition.visible = true if _transition.has_method("transition_out"): await _transition.transition_out() else: await get_tree().create_timer(0.3).timeout func _play_transition_in() -> void: if not _transition: transition_finished.emit() return if _transition.has_method("transition_in"): await _transition.transition_in() else: await get_tree().create_timer(0.3).timeout _transition.visible = false transition_finished.emit() ``` ### Pattern 7: Save System ```gdscript # save_manager.gd (Autoload) extends Node const SAVE_PATH := "user://savegame.save" const ENCRYPTION_KEY := "your_secret_key_here" signal save_completed signal load_completed signal save_error(message: String) func save_game(data: Dictionary) -> void: var file := FileAccess.open_encrypted_with_pass( SAVE_PATH, FileAccess.WRITE, ENCRYPTION_KEY ) if file == null: save_error.emit("Could not open save file") return var json := JSON.stringify(data) file.store_string(json) file.close() save_completed.emit() func load_game() -> Dictionary: if not FileAccess.file_exists(SAVE_PATH): return {} var file := FileAccess.open_encrypted_with_pass( SAVE_PATH, FileAccess.READ, ENCRYPTION_KEY ) if file == null: save_error.emit("Could not open save file") return {} var json := file.get_as_text() file.close() var parsed := JSON.parse_string(json) if parsed == null: save_error.emit("Could not parse save data") return {} load_completed.emit() return parsed func delete_save() -> void: if FileAccess.file_exists(SAVE_PATH): DirAccess.remove_absolute(SAVE_PATH) func has_save() -> bool: return FileAccess.file_exists(SAVE_PATH) ``` ```gdscript # saveable.gd (Attach to saveable nodes) class_name Saveable extends Node @export var save_id: String func _ready() -> void: if save_id.is_empty(): save_id = str(get_path()) func get_save_data() -> Dictionary: var parent := get_parent() var data := {"id": save_id} if parent is Node2D: data["position"] = {"x": parent.position.x, "y": parent.position.y} if parent.has_method("get_custom_save_data"): data.merge(parent.get_custom_save_data()) return data func load_save_data(data: Dictionary) -> void: var parent := get_parent() if data.has("position") and parent is Node2D: parent.position = Vector2(data.position.x, data.position.y) if parent.has_method("load_custom_save_data"): parent.load_custom_save_data(data) ``` ## Performance Tips ```gdscript # 1. Cache node references @onready var sprite := $Sprite2D # Good # $Sprite2D in _process() # Bad - repeated lookup # 2. Use object pooling for frequent spawning # See Pattern 4 # 3. Avoid allocations in hot paths var _reusable_array: Array = [] func _process(_delta: float) -> void: _reusable_array.clear() # Reuse instead of creating new # 4. Use static typing func calculate(value: float) -> float: # Good return value * 2.0 # 5. Disable processing when not needed func _on_off_screen() -> void: set_process(false) set_physics_process(false) ``` ## Best Practices ### Do's - **Use signals for decoupling** - Avoid direct references - **Type everything** - Static typing catches errors - **Use resources for data** - Separate data from logic - **Pool frequently spawned objects** - Avoid GC hitches - **Use Autoloads sparingly** - Only for truly global systems ### Don'ts - **Don't use `get_node()` in loops** - Cache ref
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

embedding-strategies

Select and optimize embedding models for semantic search and RAG

coding
⭐1
# Embedding Strategies Guide to selecting and optimizing embedding models for vector search applications. ## When to Use This Skill - Choosing embedding models for RAG - Optimizing chunking strategies - Fine-tuning embeddings for domains - Comparing embedding model performance - Reducing embedding dimensions - Handling multilingual content ## Core Concepts ### 1. Embedding Model Comparison (2026) | Model | Dimensions | Max Tokens | Best For | | -------------------------- | ---------- | ---------- | ----------------------------------- | | **voyage-3-large** | 1024 | 32000 | Claude apps (Anthropic recommended) | | **voyage-3** | 1024 | 32000 | Claude apps, cost-effective | | **voyage-code-3** | 1024 | 32000 | Code search | | **voyage-finance-2** | 1024 | 32000 | Financial documents | | **voyage-law-2** | 1024 | 32000 | Legal documents | | **text-embedding-3-large** | 3072 | 8191 | OpenAI apps, high accuracy | | **text-embedding-3-small** | 1536 | 8191 | OpenAI apps, cost-effective | | **bge-large-en-v1.5** | 1024 | 512 | Open source, local deployment | | **all-MiniLM-L6-v2** | 384 | 256 | Fast, lightweight | | **multilingual-e5-large** | 1024 | 512 | Multi-language | ### 2. Embedding Pipeline ``` Document → Chunking → Preprocessing → Embedding Model → Vector ↓ [Overlap, Size] [Clean, Normalize] [API/Local] ``` ## Templates ### Template 1: Voyage AI Embeddings (Recommended for Claude) ```python from langchain_voyageai import VoyageAIEmbeddings from typing import List import os # Initialize Voyage AI embeddings (recommended by Anthropic for Claude) embeddings = VoyageAIEmbeddings( model="voyage-3-large", voyage_api_key=os.environ.get("VOYAGE_API_KEY") ) def get_embeddings(texts: List[str]) -> List[List[float]]: """Get embeddings from Voyage AI.""" return embeddings.embed_documents(texts) def get_query_embedding(query: str) -> List[float]: """Get single query embedding.""" return embeddings.embed_query(query) # Specialized models for domains code_embeddings = VoyageAIEmbeddings(model="voyage-code-3") finance_embeddings = VoyageAIEmbeddings(model="voyage-finance-2") legal_embeddings = VoyageAIEmbeddings(model="voyage-law-2") ``` ### Template 2: OpenAI Embeddings ```python from openai import OpenAI from typing import List import numpy as np client = OpenAI() def get_embeddings( texts: List[str], model: str = "text-embedding-3-small", dimensions: int = None ) -> List[List[float]]: """Get embeddings from OpenAI with optional dimension reduction.""" # Handle batching for large lists batch_size = 100 all_embeddings = [] for i in range(0, len(texts), batch_size): batch = texts[i:i + batch_size] kwargs = {"input": batch, "model": model} if dimensions: # Matryoshka dimensionality reduction kwargs["dimensions"] = dimensions response = client.embeddings.create(**kwargs) embeddings = [item.embedding for item in response.data] all_embeddings.extend(embeddings) return all_embeddings def get_embedding(text: str, **kwargs) -> List[float]: """Get single embedding.""" return get_embeddings([text], **kwargs)[0] # Dimension reduction with Matryoshka embeddings def get_reduced_embedding(text: str, dimensions: int = 512) -> List[float]: """Get embedding with reduced dimensions (Matryoshka).""" return get_embedding( text, model="text-embedding-3-small", dimensions=dimensions ) ``` ### Template 3: Local Embeddings with Sentence Transformers ```python from sentence_transformers import SentenceTransformer from typing import List, Optional import numpy as np class LocalEmbedder: """Local embedding with sentence-transformers.""" def __init__( self, model_name: str = "BAAI/bge-large-en-v1.5", device: str = "cuda" ): self.model = SentenceTransformer(model_name, device=device) self.model_name = model_name def embed( self, texts: List[str], normalize: bool = True, show_progress: bool = False ) -> np.ndarray: """Embed texts with optional normalization.""" embeddings = self.model.encode( texts, normalize_embeddings=normalize, show_progress_bar=show_progress, convert_to_numpy=True ) return embeddings def embed_query(self, query: str) -> np.ndarray: """Embed a query with appropriate prefix for retrieval models.""" # BGE and similar models benefit from query prefix if "bge" in self.model_name.lower(): query = f"Represent this sentence for searching relevant passages: {query}" return self.embed([query])[0] def embed_documents(self, documents: List[str]) -> np.ndarray: """Embed documents for indexing.""" return self.embed(documents) # E5 model with instructions class E5Embedder: def __init__(self, model_name: str = "intfloat/multilingual-e5-large"): self.model = SentenceTransformer(model_name) def embed_query(self, query: str) -> np.ndarray: """E5 requires 'query:' prefix for queries.""" return self.model.encode(f"query: {query}") def embed_document(self, document: str) -> np.ndarray: """E5 requires 'passage:' prefix for documents.""" return self.model.encode(f"passage: {document}") ``` ### Template 4: Chunking Strategies ```python from typing import List, Tuple import re def chunk_by_tokens( text: str, chunk_size: int = 512, chunk_overlap: int = 50, tokenizer=None ) -> List[str]: """Chunk text by token count.""" import tiktoken tokenizer = tokenizer or tiktoken.get_encoding("cl100k_base") tokens = tokenizer.encode(text) chunks = [] start = 0 while start < len(tokens): end = start + chunk_size chunk_tokens = tokens[start:end] chunk_text = tokenizer.decode(chunk_tokens) chunks.append(chunk_text) start = end - chunk_overlap return chunks def chunk_by_sentences( text: str, max_chunk_size: int = 1000, min_chunk_size: int = 100 ) -> List[str]: """Chunk text by sentences, respecting size limits.""" import nltk sentences = nltk.sent_tokenize(text) chunks = [] current_chunk = [] current_size = 0 for sentence in sentences: sentence_size = len(sentence) if current_size + sentence_size > max_chunk_size and current_chunk: chunks.append(" ".join(current_chunk)) current_chunk = [] current_size = 0 current_chunk.append(sentence) current_size += sentence_size if current_chunk: chunks.append(" ".join(current_chunk)) return chunks def chunk_by_semantic_sections( text: str, headers_pattern: str = r'^#{1,3}\s+.+$' ) -> List[Tuple[str, str]]: """Chunk markdown by headers, preserving hierarchy.""" lines = text.split('\n') chunks = [] current_header = "" current_content = [] for line in lines: if re.match(headers_pattern, line, re.MULTILINE): if current_content: chunks.append((current_header, '\n'.join(current_content))) current_header = line current_content = [] else: current_content.append(line) if current_content: chunks.append((current_header, '\n'.join(current_content))) return chunks def recursive_character_splitter( text: str, chunk_size: int = 1000, chunk_overlap: int = 200, separators: List[str] = None ) -> List[str]: """LangChain-style recursive splitter.""" separators = separators or ["\n\n", "\n", ". ", " ", ""] def split_text(text: str, separators: List[str]) -> List[str]: if not text: return [] separator = separators[0] remaining_separators = separators[1:] if separator == "": # Character-level split return [text[i:i+chunk_size] for i in range(0, len(text), chunk_size - chunk_overlap)] splits = text.split(separator) chunks = [] current_chunk = [] current_length = 0 for split in splits: split_length = len(split) + len(separator) if current_length + split_length > chunk_size and current_chunk: chunk_text = separator.join(current_chunk) # Recursively split if still too large if len(chunk_text) > chunk_size and remaining_separators: chunks.extend(split_text(chunk_text, remaining_separators)) else: chunks.append(chunk_text) # Start new chunk with overlap overlap_splits = [] overlap_length = 0 for s in reversed(current_chunk): if overlap_length + len(s) <= chunk_overlap: overlap_splits.insert(0, s) overlap_length += len(s) else: break current_chunk = overlap_splits current_length = overlap_length current_chunk.append(split) current_length += split_length if current_chunk: chunks.append(separator.join(current_chunk)) return chunks return split_text(text, separators) ``` ### Template 5: Domain-Specific Embedding Pipeline ```python import re from typing import List, Optional from dataclasses import dataclass @dataclass class EmbeddedDocument: id: str document_id: str chunk_index: int text: str embedding: List[float] metadata: dict class DomainEmbeddingPipeline: """Pipeline for domain-specific embeddings.""" def __init__( self, embedding_model: str = "voyage-3-large", chunk_size: int = 512, chunk_overlap: int = 50, preprocessing_fn=None ): self.embeddings = VoyageAIEmbeddings(model=embedding_model) self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap self.preprocess = preprocessing_fn or self._default_preprocess def _default_preprocess(self, text: str) -> str: """Default preprocessing.""" # Remove excessive whitespace text = re.sub(r'\s+', ' ', text) # Remove special characters (customize for your domain) text = re.sub(r'[^\w\s.,!?-]', '', text) return text.strip() async def process_documents( self, documents: List[dict], id_field: str = "id", content_field: str = "content", metadata_fields: Optional[List[str]] = None ) -> List[EmbeddedDocument]: """Process documents for vector storage.""" processed = [] for doc in documents: content = doc[content_field] doc_id = doc[id_field] # Preprocess cleaned = self.preprocess(content) # Chunk chunks = chunk_by_tokens( cleaned, self.chunk_size, self.chunk_overlap ) # Create embeddings embeddings = await self.embeddings.aembed_documents(chunks) # Create records for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)): metadata = {"document_id": doc_id, "chunk_index": i} # Add specified metadata fields if metadata_fields: for field in metadata_fields: if field in doc: metadata[field] = doc[field] processed.append(EmbeddedDocument( id=f"{doc_id}_chunk_{i}", document_id=doc_id, chunk_index=i, text=chunk, embedding=embedding, metadata=metadata )) return processed # Code-specific pipeline class CodeEmbeddingPipeline: """Specialized pipeline for code embeddings.""" def __init__(self): # Use Voyage's code-specific model self.embeddings = VoyageAIEmbeddings(model="voyage-code-3") def chunk_code(self, code: str, language: str) -> List[dict]: """Chunk code by functions/classes using tree-sitter.""" try: import tree_sitter_languages parser = tree_sitter_languages.get_parser(language) tree = parser.parse(bytes(code, "utf8")) chunks = [] # Extract function and class definitions self._extract_nodes(tree.root_node, code, chunks) return chunks except ImportError: # Fallback to simple chunking return [{"text": code, "type": "module"}] def _extract_nodes(self, node, source_code: str, chunks: list): """Recursively extract function/class definitions.""" if node.type in ['function_definition', 'class_definition', 'method_definition']: text = source_code[node.start_byte:node.end_byte] chunks.append({ "text": text, "type": node.type, "name": self._get_name(node), "start_line": node.start_point[0], "end_line": node.end_point[0] }) for child in node.children: self._extract_nodes(child, source_code, chunks) def _get_name(self, node) -> str: """Extract name from function/class node.""" for child in node.children: if child.type == 'identifier' or child.type == 'name': return child.text.decode('utf8') return "unknown" async def embed_with_context( self, chunk: str, context: str = "" ) -> List[float]: """Embed code with surrounding context.""" if context: combined = f"Context: {context}\n\nCode:\n{chunk}" else: combined = chunk return await self.embeddings.aembed_query(combined) ``` ### Template 6: Embedding Quality Evaluation ```python import numpy as np from typing import List, Dict def evaluate_retrieval_quality( queries: List[str], relevant_docs: List[List[str]], # List of relevant doc IDs per query retrieved_docs: List[List[str]], # List of retrieved doc IDs per query k: int = 10 ) -> Dict[str, float]: """Evaluate embedding quality for retrieval.""" def precision_at_k(relevant: set, retrieved: List[str], k: int) -> float: retrieved_k = retrieved[:k] relevant_retrieved = len(set(retrieved_k) & relevant) return relevant_retrieved / k if k > 0 else 0 def recall_at_k(relevant: set, retrieved: List[str], k: int) -> float: retrieved_k = retrieved[:k] relevant_retrieved = len(set(retrieved_k) & relevant) return relevant_retrieved / len(relevant) if relevant else 0 def mrr(relevant: set, retrieved: List[str]) -> float: for i, doc in enumerate(retrieved): if doc in relevant: return 1 / (i + 1) return 0 def ndcg_at_k(relevant: set, retrieved: List[str], k: int) -> float: dcg = sum( 1 / np.log2(i + 2) if doc in relevant else 0 for i, doc in enumerate(retrieved[:k]) ) ideal_dcg = sum(1 / np.log2(i + 2) for i in range(min(len(relevant), k))) return dcg / ideal_dcg if ideal_dcg > 0 else 0 metrics = { f"precision@{k}": [], f"recall@{k}": [], "mrr": [], f"ndcg@{k}": [] } for relevant, retrieved in zip(relevant_docs, retrieved_docs): relevant_set = set(relevant) metrics[f"precision@{k}"].append(precision_at_k(relevant_set, retrieved, k)) metrics[f"recall@{k}"].append(recall_at_k(relevant_set, retrieved, k)) metrics["mrr"].append(mrr(relevant_set, retrieved)) metrics[f"ndcg@{k}"].append(ndcg_at_k(relevant_set, retrieved, k)) return {name: np.mean(values) for name, values in metrics.items()} def compute_embedding_similarity( embeddings1: np.ndarray, embeddings2: np.ndarray, metric: str = "cosine" ) -> np.ndarray: """Compute similarity matrix between embedding sets.""" if metric == "cosine": # Normalize and compute dot product norm1 = embeddings1 / np.linalg.norm(embeddings1, axis=1, keepdims=True) norm2 = embeddings2 / np.linalg.norm(embeddings2, axis=1, keepdims=True) return norm1 @ norm2.T elif metric == "euclidean": from scipy.spatial.distance import cdist return -cdist(embeddings1, embeddings2, metric='euclidean') elif metric == "dot": return embeddings1 @ embeddings2.T else: raise ValueError(f"Unknown metric: {metric}") def compare_embedding_models( texts: List[str], models: Dict[str, callable], queries: List[str], relevant_indices: List[List[int]], k: int = 5 ) -> Dict[str, Dict[str, float]]: """Compare multiple embedding models on retrieval quality.""" results = {} for model_name, embed_fn in models.items(): # Embed all texts doc_embeddings = np.array(embed_fn(texts)) retrieved_per_query = [] for query in queries: query_embedding = np.array(embed_fn([query])[0]) # Compute similarities similarities = compute_embedding_similarity( query_embedding.reshape(1, -1), doc_embeddings, metric="cosine" )[0] # Get top-k indices top_k_indices = np.argsort(similarities)[::-1][:k] retrieved_per_query.append([str(i) for i in top_k_indices]) # Convert relevant indices to string IDs relevant_docs = [[str(i) for i in indices] for indices in relevant_indices] results[model_name] = evaluate_retrieval_quality( queries, relevant_docs, retrieved_per_query, k ) return results ``` ## Best Practices ### Do's - **Match model to use case**: Code vs prose vs multilingual - **Chunk thoughtfully**: Preserve semantic boundaries - **Normalize embeddings**: For cosine similarity search - **Batch requests**: More efficient than one-by-one - **Cache embeddings**: Avoid recomputing for static content - **Use Voyage AI for Claude apps**: Recommended by Anthropic ### Don'ts - **Don't ignore token limits**: Truncation loses information - **Don't mix embedding models**: Incompatible vector spaces - **Don't skip preprocessing**: Garbage in, garbage out - **Don't over-chunk**: Lose important context - **Don't forget metadata**: Essential for filtering and debugging ## Resources - [Voyage AI Documentation](https://docs.voyageai.com/) - [OpenAI Embeddings Guide](https://platform.openai.com/docs/guides/embeddings) - [Sentence Transformers](https://www.sbert.net/) - [MTEB Benchmar
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

langchain-architecture

Design LLM applications using LangChain 1.x and LangGraph for

coding
⭐1
# LangChain & LangGraph Architecture Master modern LangChain 1.x and LangGraph for building sophisticated LLM applications with agents, state management, memory, and tool integration. ## When to Use This Skill - Building autonomous AI agents with tool access - Implementing complex multi-step LLM workflows - Managing conversation memory and state - Integrating LLMs with external data sources and APIs - Creating modular, reusable LLM application components - Implementing document processing pipelines - Building production-grade LLM applications ## Package Structure (LangChain 1.x) ``` langchain (1.2.x) # High-level orchestration langchain-core (1.2.x) # Core abstractions (messages, prompts, tools) langchain-community # Third-party integrations langgraph # Agent orchestration and state management langchain-openai # OpenAI integrations langchain-anthropic # Anthropic/Claude integrations langchain-voyageai # Voyage AI embeddings langchain-pinecone # Pinecone vector store ``` ## Core Concepts ### 1. LangGraph Agents LangGraph is the standard for building agents in 2026. It provides: **Key Features:** - **StateGraph**: Explicit state management with typed state - **Durable Execution**: Agents persist through failures - **Human-in-the-Loop**: Inspect and modify state at any point - **Memory**: Short-term and long-term memory across sessions - **Checkpointing**: Save and resume agent state **Agent Patterns:** - **ReAct**: Reasoning + Acting with `create_react_agent` - **Plan-and-Execute**: Separate planning and execution nodes - **Multi-Agent**: Supervisor routing between specialized agents - **Tool-Calling**: Structured tool invocation with Pydantic schemas ### 2. State Management LangGraph uses TypedDict for explicit state: ```python from typing import Annotated, TypedDict from langgraph.graph import MessagesState # Simple message-based state class AgentState(MessagesState): """Extends MessagesState with custom fields.""" context: Annotated[list, "retrieved documents"] # Custom state for complex agents class CustomState(TypedDict): messages: Annotated[list, "conversation history"] context: Annotated[dict, "retrieved context"] current_step: str results: list ``` ### 3. Memory Systems Modern memory implementations: - **ConversationBufferMemory**: Stores all messages (short conversations) - **ConversationSummaryMemory**: Summarizes older messages (long conversations) - **ConversationTokenBufferMemory**: Token-based windowing - **VectorStoreRetrieverMemory**: Semantic similarity retrieval - **LangGraph Checkpointers**: Persistent state across sessions ### 4. Document Processing Loading, transforming, and storing documents: **Components:** - **Document Loaders**: Load from various sources - **Text Splitters**: Chunk documents intelligently - **Vector Stores**: Store and retrieve embeddings - **Retrievers**: Fetch relevant documents ### 5. Callbacks & Tracing LangSmith is the standard for observability: - Request/response logging - Token usage tracking - Latency monitoring - Error tracking - Trace visualization ## Quick Start ### Modern ReAct Agent with LangGraph ```python from langgraph.prebuilt import create_react_agent from langgraph.checkpoint.memory import MemorySaver from langchain_anthropic import ChatAnthropic from langchain_core.tools import tool import ast import operator # Initialize LLM (Claude Sonnet 4.6 recommended) llm = ChatAnthropic(model="claude-sonnet-4-6", temperature=0) # Define tools with Pydantic schemas @tool def search_database(query: str) -> str: """Search internal database for information.""" # Your database search logic return f"Results for: {query}" @tool def calculate(expression: str) -> str: """Safely evaluate a mathematical expression. Supports: +, -, *, /, **, %, parentheses Example: '(2 + 3) * 4' returns '20' """ # Safe math evaluation using ast allowed_operators = { ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.Pow: operator.pow, ast.Mod: operator.mod, ast.USub: operator.neg, } def _eval(node): if isinstance(node, ast.Constant): return node.value elif isinstance(node, ast.BinOp): left = _eval(node.left) right = _eval(node.right) return allowed_operators[type(node.op)](left, right) elif isinstance(node, ast.UnaryOp): operand = _eval(node.operand) return allowed_operators[type(node.op)](operand) else: raise ValueError(f"Unsupported operation: {type(node)}") try: tree = ast.parse(expression, mode='eval') return str(_eval(tree.body)) except Exception as e: return f"Error: {e}" tools = [search_database, calculate] # Create checkpointer for memory persistence checkpointer = MemorySaver() # Create ReAct agent agent = create_react_agent( llm, tools, checkpointer=checkpointer ) # Run agent with thread ID for memory config = {"configurable": {"thread_id": "user-123"}} result = await agent.ainvoke( {"messages": [("user", "Search for Python tutorials and calculate 25 * 4")]}, config=config ) ``` ## Architecture Patterns ### Pattern 1: RAG with LangGraph ```python from langgraph.graph import StateGraph, START, END from langchain_anthropic import ChatAnthropic from langchain_voyageai import VoyageAIEmbeddings from langchain_pinecone import PineconeVectorStore from langchain_core.documents import Document from langchain_core.prompts import ChatPromptTemplate from typing import TypedDict, Annotated class RAGState(TypedDict): question: str context: Annotated[list[Document], "retrieved documents"] answer: str # Initialize components llm = ChatAnthropic(model="claude-sonnet-4-6") embeddings = VoyageAIEmbeddings(model="voyage-3-large") vectorstore = PineconeVectorStore(index_name="docs", embedding=embeddings) retriever = vectorstore.as_retriever(search_kwargs={"k": 4}) # Define nodes async def retrieve(state: RAGState) -> RAGState: """Retrieve relevant documents.""" docs = await retriever.ainvoke(state["question"]) return {"context": docs} async def generate(state: RAGState) -> RAGState: """Generate answer from context.""" prompt = ChatPromptTemplate.from_template( """Answer based on the context below. If you cannot answer, say so. Context: {context} Question: {question} Answer:""" ) context_text = "\n\n".join(doc.page_content for doc in state["context"]) response = await llm.ainvoke( prompt.format(context=context_text, question=state["question"]) ) return {"answer": response.content} # Build graph builder = StateGraph(RAGState) builder.add_node("retrieve", retrieve) builder.add_node("generate", generate) builder.add_edge(START, "retrieve") builder.add_edge("retrieve", "generate") builder.add_edge("generate", END) rag_chain = builder.compile() # Use the chain result = await rag_chain.ainvoke({"question": "What is the main topic?"}) ``` ### Pattern 2: Custom Agent with Structured Tools ```python from langchain_core.tools import StructuredTool from pydantic import BaseModel, Field class SearchInput(BaseModel): """Input for database search.""" query: str = Field(description="Search query") filters: dict = Field(default={}, description="Optional filters") class EmailInput(BaseModel): """Input for sending email.""" recipient: str = Field(description="Email recipient") subject: str = Field(description="Email subject") content: str = Field(description="Email body") async def search_database(query: str, filters: dict = {}) -> str: """Search internal database for information.""" # Your database search logic return f"Results for '{query}' with filters {filters}" async def send_email(recipient: str, subject: str, content: str) -> str: """Send an email to specified recipient.""" # Email sending logic return f"Email sent to {recipient}" tools = [ StructuredTool.from_function( coroutine=search_database, name="search_database", description="Search internal database", args_schema=SearchInput ), StructuredTool.from_function( coroutine=send_email, name="send_email", description="Send an email", args_schema=EmailInput ) ] agent = create_react_agent(llm, tools) ``` ### Pattern 3: Multi-Step Workflow with StateGraph ```python from langgraph.graph import StateGraph, START, END from typing import TypedDict, Literal class WorkflowState(TypedDict): text: str entities: list analysis: str summary: str current_step: str async def extract_entities(state: WorkflowState) -> WorkflowState: """Extract key entities from text.""" prompt = f"Extract key entities from: {state['text']}\n\nReturn as JSON list." response = await llm.ainvoke(prompt) return {"entities": response.content, "current_step": "analyze"} async def analyze_entities(state: WorkflowState) -> WorkflowState: """Analyze extracted entities.""" prompt = f"Analyze these entities: {state['entities']}\n\nProvide insights." response = await llm.ainvoke(prompt) return {"analysis": response.content, "current_step": "summarize"} async def generate_summary(state: WorkflowState) -> WorkflowState: """Generate final summary.""" prompt = f"""Summarize: Entities: {state['entities']} Analysis: {state['analysis']} Provide a concise summary.""" response = await llm.ainvoke(prompt) return {"summary": response.content, "current_step": "complete"} def route_step(state: WorkflowState) -> Literal["analyze", "summarize", "end"]: """Route to next step based on current state.""" step = state.get("current_step", "extract") if step == "analyze": return "analyze" elif step == "summarize": return "summarize" return "end" # Build workflow builder = StateGraph(WorkflowState) builder.add_node("extract", extract_entities) builder.add_node("analyze", analyze_entities) builder.add_node("summarize", generate_summary) builder.add_edge(START, "extract") builder.add_conditional_edges("extract", route_step, { "analyze": "analyze", "summarize": "summarize", "end": END }) builder.add_conditional_edges("analyze", route_step, { "summarize": "summarize", "end": END }) builder.add_edge("summarize", END) workflow = builder.compile() ``` ### Pattern 4: Multi-Agent Orchestration ```python from langgraph.graph import StateGraph, START, END from langgraph.prebuilt import create_react_agent from langchain_core.messages import HumanMessage from typing import Literal class MultiAgentState(TypedDict): messages: list next_agent: str # Create specialized agents researcher = create_react_agent(llm, research_tools) writer = create_react_agent(llm, writing_tools) reviewer = create_react_agent(llm, review_tools) async def supervisor(state: MultiAgentState) -> MultiAgentState: """Route to appropriate agent based on task.""" prompt = f"""Based on the conversation, which agent should handle this? Options: - researcher: For finding information - writer: For creating content - reviewer: For reviewing and editing - FINISH: Task is complete Messages: {state['messages']} Respond with just the agent name.""" response = await llm.ainvoke(prompt) return {"next_agent": response.content.strip().lower()} def route_to_agent(state: MultiAgentState) -> Literal["researcher", "writer", "reviewer", "end"]: """Route based on supervisor decision.""" next_agent = state.get("next_agent", "").lower() if next_agent == "finish": return "end" return next_agent if next_agent in ["researcher", "writer", "reviewer"] else "end" # Build multi-agent graph builder = StateGraph(MultiAgentState) builder.add_node("supervisor", supervisor) builder.add_node("researcher", researcher) builder.add_node("writer", writer) builder.add_node("reviewer", reviewer) builder.add_edge(START, "supervisor") builder.add_conditional_edges("supervisor", route_to_agent, { "researcher": "researcher", "writer": "writer", "reviewer": "reviewer", "end": END }) # Each agent returns to supervisor for agent in ["researcher", "writer", "reviewer"]: builder.add_edge(agent, "supervisor") multi_agent = builder.compile() ``` ## Memory Management ### Token-Based Memory with LangGraph ```python from langgraph.checkpoint.memory import MemorySaver from langgraph.prebuilt import create_react_agent # In-memory checkpointer (development) checkpointer = MemorySaver() # Create agent with persistent memory agent = create_react_agent(llm, tools, checkpointer=checkpointer) # Each thread_id maintains separate conversation config = {"configurable": {"thread_id": "session-abc123"}} # Messages persist across invocations with same thread_id result1 = await agent.ainvoke({"messages": [("user", "My name is Alice")]}, config) result2 = await agent.ainvoke({"messages": [("user", "What's my name?")]}, config) # Agent remembers: "Your name is Alice" ``` ### Production Memory with PostgreSQL ```python from langgraph.checkpoint.postgres import PostgresSaver # Production checkpointer checkpointer = PostgresSaver.from_conn_string( "postgresql://user:pass@localhost/langgraph" ) agent = create_react_agent(llm, tools, checkpointer=checkpointer) ``` ### Vector Store Memory for Long-Term Context ```python from langchain_community.vectorstores import Chroma from langchain_voyageai import VoyageAIEmbeddings embeddings = VoyageAIEmbeddings(model="voyage-3-large") memory_store = Chroma( collection_name="conversation_memory", embedding_function=embeddings, persist_directory="./memory_db" ) async def retrieve_relevant_memory(query: str, k: int = 5) -> list: """Retrieve relevant past conversations.""" docs = await memory_store.asimilarity_search(query, k=k) return [doc.page_content for doc in docs] async def store_memory(content: str, metadata: dict = {}): """Store conversation in long-term memory.""" await memory_store.aadd_texts([content], metadatas=[metadata]) ``` ## Callback System & LangSmith ### LangSmith Tracing ```python import os from langchain_anthropic import ChatAnthropic # Enable LangSmith tracing os.environ["LANGCHAIN_TRACING_V2"] = "true" os.environ["LANGCHAIN_API_KEY"] = "your-api-key" os.environ["LANGCHAIN_PROJECT"] = "my-project" # All LangChain/LangGraph operations are automatically traced llm = ChatAnthropic(model="claude-sonnet-4-6") ``` ### Custom Callback Handler ```python from langchain_core.callbacks import BaseCallbackHandler from typing import Any, Dict, List class CustomCallbackHandler(BaseCallbackHandler): def on_llm_start( self, serialized: Dict[str, Any], prompts: List[str], **kwargs ) -> None: print(f"LLM started with {len(prompts)} prompts") def on_llm_end(self, response, **kwargs) -> None: print(f"LLM completed: {len(response.generations)} generations") def on_llm_error(self, error: Exception, **kwargs) -> None: print(f"LLM error: {error}") def on_tool_start( self, serialized: Dict[str, Any], input_str: str, **kwargs ) -> None: print(f"Tool started: {serialized.get('name')}") def on_tool_end(self, output: str, **kwargs) -> None: print(f"Tool completed: {output[:100]}...") # Use callbacks result = await agent.ainvoke( {"messages": [("user", "query")]}, config={"callbacks": [CustomCallbackHandler()]} ) ``` ## Streaming Responses ```python from langchain_anthropic import ChatAnthropic llm = ChatAnthropic(model="claude-sonnet-4-6", streaming=True) # Stream tokens async for chunk in llm.astream("Tell me a story"): print(chunk.content, end="", flush=True) # Stream agent events async for event in agent.astream_events( {"messages": [("user", "Search and summarize")]}, version="v2" ): if event["event"] == "on_chat_model_stream": print(event["data"]["chunk"].content, end="") elif event["event"] == "on_tool_start": print(f"\n[Using tool: {event['name']}]") ``` ## Testing Strategies ```python import pytest from unittest.mock import AsyncMock, patch @pytest.mark.asyncio async def test_agent_tool_selection(): """Test agent selects correct tool.""" with patch.object(llm, 'ainvoke') as mock_llm: mock_llm.return_value = AsyncMock(content="Using search_database") result = await agent.ainvoke({ "messages": [("user", "search for documents")] }) # Verify tool was called assert "search_database" in str(result) @pytest.mark.asyncio async def test_memory_persistence(): """Test memory persists across invocations.""" config = {"configurable": {"thread_id": "test-thread"}} # First message await agent.ainvoke( {"messages": [("user", "Remember: the code is 12345")]}, config ) # Second message should remember result = await agent.ainvoke( {"messages": [("user", "What was the code?")]}, config ) assert "12345" in result["messages"][-1].content ``` ## Performance Optimization ### 1. Caching with Redis ```python from langchain_community.cache import RedisCache from langchain_core.globals import set_llm_cache import redis redis_client = redis.Redis.from_url("redis://localhost:6379") set_llm_cache(RedisCache(redis_client)) ``` ### 2. Async Batch Processing ```python import asyncio from langchain_core.documents import Document async def process_documents(documents: list[Document]) -> list: """Process documents in parallel.""" tasks = [process_single(doc) for doc in documents] return await asyncio.gather(*tasks) async def process_single(doc: Document) -> dict: """Process a single document.""" chunks = text_splitter.split_documents([doc]) embeddings = await embeddings_model.aembed_documents( [c.page_content for c in chunks] ) return {"doc_id": doc.metadata.get("id"), "embeddings": embeddings} ``` ### 3. Connection Pooling ```python from langchain_pinecone import PineconeVectorStore from pinecone import Pinecone # Reuse Pinecone client pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"]) index = pc.Index("my-index") # Create vector store with existing index vectorstore = PineconeVectorStore(index=index, embedding=embeddings) ``` ## Resources - [LangChain Documentation](https://python.langchain.com/docs/) - [LangGraph Documentation](https://langchain-ai.github.io/langgraph/) - [LangSmith Platform](https://smith.langchain.com/) - [LangChain GitHub](https://github.com/langchain-ai/langchain) - [LangGraph GitHub](https://github.com/langchain-ai/langgraph) ## Common Pitfalls 1. **Using Deprecated APIs**: Use LangGraph for agents, not `initialize_agent` 2. **Memory Overflow**: Use checkpointers with TTL for long-running agents 3. **Poor Tool Descriptions**: Clear descriptions help LLM select correct tools 4. **Context Window Exceeded**: Use summarization or sliding window memory 5. **No Error Handling**: Wrap too
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

prompt-engineering-patterns

Master advanced prompt engineering techniques to maximize LLM

coding
⭐1
# Prompt Engineering Patterns Master advanced prompt engineering techniques to maximize LLM performance, reliability, and controllability. ## When to Use This Skill - Designing complex prompts for production LLM applications - Optimizing prompt performance and consistency - Implementing structured reasoning patterns (chain-of-thought, tree-of-thought) - Building few-shot learning systems with dynamic example selection - Creating reusable prompt templates with variable interpolation - Debugging and refining prompts that produce inconsistent outputs - Implementing system prompts for specialized AI assistants - Using structured outputs (JSON mode) for reliable parsing ## Core Capabilities ### 1. Few-Shot Learning - Example selection strategies (semantic similarity, diversity sampling) - Balancing example count with context window constraints - Constructing effective demonstrations with input-output pairs - Dynamic example retrieval from knowledge bases - Handling edge cases through strategic example selection ### 2. Chain-of-Thought Prompting - Step-by-step reasoning elicitation - Zero-shot CoT with "Let's think step by step" - Few-shot CoT with reasoning traces - Self-consistency techniques (sampling multiple reasoning paths) - Verification and validation steps ### 3. Structured Outputs - JSON mode for reliable parsing - Pydantic schema enforcement - Type-safe response handling - Error handling for malformed outputs ### 4. Prompt Optimization - Iterative refinement workflows - A/B testing prompt variations - Measuring prompt performance metrics (accuracy, consistency, latency) - Reducing token usage while maintaining quality - Handling edge cases and failure modes ### 5. Template Systems - Variable interpolation and formatting - Conditional prompt sections - Multi-turn conversation templates - Role-based prompt composition - Modular prompt components ### 6. System Prompt Design - Setting model behavior and constraints - Defining output formats and structure - Establishing role and expertise - Safety guidelines and content policies - Context setting and background information ## Quick Start ```python from langchain_anthropic import ChatAnthropic from langchain_core.prompts import ChatPromptTemplate from pydantic import BaseModel, Field # Define structured output schema class SQLQuery(BaseModel): query: str = Field(description="The SQL query") explanation: str = Field(description="Brief explanation of what the query does") tables_used: list[str] = Field(description="List of tables referenced") # Initialize model with structured output llm = ChatAnthropic(model="claude-sonnet-4-6") structured_llm = llm.with_structured_output(SQLQuery) # Create prompt template prompt = ChatPromptTemplate.from_messages([ ("system", """You are an expert SQL developer. Generate efficient, secure SQL queries. Always use parameterized queries to prevent SQL injection. Explain your reasoning briefly."""), ("user", "Convert this to SQL: {query}") ]) # Create chain chain = prompt | structured_llm # Use result = await chain.ainvoke({ "query": "Find all users who registered in the last 30 days" }) print(result.query) print(result.explanation) ``` ## Key Patterns ### Pattern 1: Structured Output with Pydantic ```python from anthropic import Anthropic from pydantic import BaseModel, Field from typing import Literal import json class SentimentAnalysis(BaseModel): sentiment: Literal["positive", "negative", "neutral"] confidence: float = Field(ge=0, le=1) key_phrases: list[str] reasoning: str async def analyze_sentiment(text: str) -> SentimentAnalysis: """Analyze sentiment with structured output.""" client = Anthropic() message = client.messages.create( model="claude-sonnet-4-6", max_tokens=500, messages=[{ "role": "user", "content": f"""Analyze the sentiment of this text. Text: {text} Respond with JSON matching this schema: {{ "sentiment": "positive" | "negative" | "neutral", "confidence": 0.0-1.0, "key_phrases": ["phrase1", "phrase2"], "reasoning": "brief explanation" }}""" }] ) return SentimentAnalysis(**json.loads(message.content[0].text)) ``` ### Pattern 2: Chain-of-Thought with Self-Verification ```python from langchain_core.prompts import ChatPromptTemplate cot_prompt = ChatPromptTemplate.from_template(""" Solve this problem step by step. Problem: {problem} Instructions: 1. Break down the problem into clear steps 2. Work through each step showing your reasoning 3. State your final answer 4. Verify your answer by checking it against the original problem Format your response as: ## Steps [Your step-by-step reasoning] ## Answer [Your final answer] ## Verification [Check that your answer is correct] """) ``` ### Pattern 3: Few-Shot with Dynamic Example Selection ```python from langchain_voyageai import VoyageAIEmbeddings from langchain_core.example_selectors import SemanticSimilarityExampleSelector from langchain_chroma import Chroma # Create example selector with semantic similarity example_selector = SemanticSimilarityExampleSelector.from_examples( examples=[ {"input": "How do I reset my password?", "output": "Go to Settings > Security > Reset Password"}, {"input": "Where can I see my order history?", "output": "Navigate to Account > Orders"}, {"input": "How do I contact support?", "output": "Click Help > Contact Us or email support@example.com"}, ], embeddings=VoyageAIEmbeddings(model="voyage-3-large"), vectorstore_cls=Chroma, k=2 # Select 2 most similar examples ) async def get_few_shot_prompt(query: str) -> str: """Build prompt with dynamically selected examples.""" examples = await example_selector.aselect_examples({"input": query}) examples_text = "\n".join( f"User: {ex['input']}\nAssistant: {ex['output']}" for ex in examples ) return f"""You are a helpful customer support assistant. Here are some example interactions: {examples_text} Now respond to this query: User: {query} Assistant:""" ``` ### Pattern 4: Progressive Disclosure Start with simple prompts, add complexity only when needed: ```python PROMPT_LEVELS = { # Level 1: Direct instruction "simple": "Summarize this article: {text}", # Level 2: Add constraints "constrained": """Summarize this article in 3 bullet points, focusing on: - Key findings - Main conclusions - Practical implications Article: {text}""", # Level 3: Add reasoning "reasoning": """Read this article carefully. 1. First, identify the main topic and thesis 2. Then, extract the key supporting points 3. Finally, summarize in 3 bullet points Article: {text} Summary:""", # Level 4: Add examples "few_shot": """Read articles and provide concise summaries. Example: Article: "New research shows that regular exercise can reduce anxiety by up to 40%..." Summary: • Regular exercise reduces anxiety by up to 40% • 30 minutes of moderate activity 3x/week is sufficient • Benefits appear within 2 weeks of starting Now summarize this article: Article: {text} Summary:""" } ``` ### Pattern 5: Error Recovery and Fallback ```python from pydantic import BaseModel, ValidationError import json class ResponseWithConfidence(BaseModel): answer: str confidence: float sources: list[str] alternative_interpretations: list[str] = [] ERROR_RECOVERY_PROMPT = """ Answer the question based on the context provided. Context: {context} Question: {question} Instructions: 1. If you can answer confidently (>0.8), provide a direct answer 2. If you're somewhat confident (0.5-0.8), provide your best answer with caveats 3. If you're uncertain (<0.5), explain what information is missing 4. Always provide alternative interpretations if the question is ambiguous Respond in JSON: {{ "answer": "your answer or 'I cannot determine this from the context'", "confidence": 0.0-1.0, "sources": ["relevant context excerpts"], "alternative_interpretations": ["if question is ambiguous"] }} """ async def answer_with_fallback( context: str, question: str, llm ) -> ResponseWithConfidence: """Answer with error recovery and fallback.""" prompt = ERROR_RECOVERY_PROMPT.format(context=context, question=question) try: response = await llm.ainvoke(prompt) return ResponseWithConfidence(**json.loads(response.content)) except (json.JSONDecodeError, ValidationError) as e: # Fallback: try to extract answer without structure simple_prompt = f"Based on: {context}\n\nAnswer: {question}" simple_response = await llm.ainvoke(simple_prompt) return ResponseWithConfidence( answer=simple_response.content, confidence=0.5, sources=["fallback extraction"], alternative_interpretations=[] ) ``` ### Pattern 6: Role-Based System Prompts ```python SYSTEM_PROMPTS = { "analyst": """You are a senior data analyst with expertise in SQL, Python, and business intelligence. Your responsibilities: - Write efficient, well-documented queries - Explain your analysis methodology - Highlight key insights and recommendations - Flag any data quality concerns Communication style: - Be precise and technical when discussing methodology - Translate technical findings into business impact - Use clear visualizations when helpful""", "assistant": """You are a helpful AI assistant focused on accuracy and clarity. Core principles: - Always cite sources when making factual claims - Acknowledge uncertainty rather than guessing - Ask clarifying questions when the request is ambiguous - Provide step-by-step explanations for complex topics Constraints: - Do not provide medical, legal, or financial advice - Redirect harmful requests appropriately - Protect user privacy""", "code_reviewer": """You are a senior software engineer conducting code reviews. Review criteria: - Correctness: Does the code work as intended? - Security: Are there any vulnerabilities? - Performance: Are there efficiency concerns? - Maintainability: Is the code readable and well-structured? - Best practices: Does it follow language idioms? Output format: 1. Summary assessment (approve/request changes) 2. Critical issues (must fix) 3. Suggestions (nice to have) 4. Positive feedback (what's done well)""" } ``` ## Integration Patterns ### With RAG Systems ```python RAG_PROMPT = """You are a knowledgeable assistant that answers questions based on provided context. Context (retrieved from knowledge base): {context} Instructions: 1. Answer ONLY based on the provided context 2. If the context doesn't contain the answer, say "I don't have information about that in my knowledge base" 3. Cite specific passages using [1], [2] notation 4. If the question is ambiguous, ask for clarification Question: {question} Answer:""" ``` ### With Validation and Verification ```python VALIDATED_PROMPT = """Complete the following task: Task: {task} After generating your response, verify it meets ALL these criteria: ✓ Directly addresses the original request ✓ Contains no factual errors ✓ Is appropriately detailed (not too brief, not too verbose) ✓ Uses proper formatting ✓ Is safe and appropriate If verification fails on any criterion, revise before responding. Response:""" ``` ## Performance Optimization ### Token Efficiency ```python # Before: Verbose prompt (150+ tokens) verbose_prompt = """ I would like you to please take the following text and provide me with a comprehensive summary of the main points. The summary should capture the key ideas and important details while being concise and easy to understand. """ # After: Concise prompt (30 tokens) concise_prompt = """Summarize the key points concisely: {text} Summary:""" ``` ### Caching Common Prefixes ```python from anthropic import Anthropic client = Anthropic() # Use prompt caching for repeated system prompts response = client.messages.create( model="claude-sonnet-4-6", max_tokens=1000, system=[ { "type": "text", "text": LONG_SYSTEM_PROMPT, "cache_control": {"type": "ephemeral"} } ], messages=[{"role": "user", "content": user_query}] ) ``` ## Best Practices 1. **Be Specific**: Vague prompts produce inconsistent results 2. **Show, Don't Tell**: Examples are more effective than descriptions 3. **Use Structured Outputs**: Enforce schemas with Pydantic for reliability 4. **Test Extensively**: Evaluate on diverse, representative inputs 5. **Iterate Rapidly**: Small changes can have large impacts 6. **Monitor Performance**: Track metrics in production 7. **Version Control**: Treat prompts as code with proper versioning 8. **Document Intent**: Explain why prompts are structured as they are ## Common Pitfalls - **Over-engineering**: Starting with complex prompts before trying simple ones - **Example pollution**: Using examples that don't match the target task - **Context overflow**: Exceeding token limits with excessive examples - **Ambiguous instructions**: Leaving room for multiple interpretations - **Ignoring edge cases**: Not testing on unusual or boundary inputs - **No error handling**: Assuming outputs will always be well-formed - **Hardcoded values**: Not parameterizing prompts for reuse ## Success Metrics Track these KPIs for your prompts: - **Accuracy**: Correctness of outputs - **Consistency**: Reproducibility across similar inputs - **Latency**: Response time (P50, P95, P99) - **Token Usage**: Average tokens per request - **Success Rate**: Percentage of valid, parseable outputs - **User Satisfaction**: Ratings and feedback ## Resources - [Anthropic Prompt Engineering Guide](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering) - [Claude Prompt Caching](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching) - [OpenAI Prompt Engineering](https://platform.openai.com/docs/guides/prompt-engineering) - [LangChain Prompts](https://python.langchain.com/docs/concepts/prompts/)
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

uv-package-manager

Master the uv package manager for fast Python dependency

coding
⭐1
# UV Package Manager Comprehensive guide to using uv, an extremely fast Python package installer and resolver written in Rust, for modern Python project management and dependency workflows. ## When to Use This Skill - Setting up new Python projects quickly - Managing Python dependencies faster than pip - Creating and managing virtual environments - Installing Python interpreters - Resolving dependency conflicts efficiently - Migrating from pip/pip-tools/poetry - Speeding up CI/CD pipelines - Managing monorepo Python projects - Working with lockfiles for reproducible builds - Optimizing Docker builds with Python dependencies ## Core Concepts ### 1. What is uv? - **Ultra-fast package installer**: 10-100x faster than pip - **Written in Rust**: Leverages Rust's performance - **Drop-in pip replacement**: Compatible with pip workflows - **Virtual environment manager**: Create and manage venvs - **Python installer**: Download and manage Python versions - **Resolver**: Advanced dependency resolution - **Lockfile support**: Reproducible installations ### 2. Key Features - Blazing fast installation speeds - Disk space efficient with global cache - Compatible with pip, pip-tools, poetry - Comprehensive dependency resolution - Cross-platform support (Linux, macOS, Windows) - No Python required for installation - Built-in virtual environment support ### 3. UV vs Traditional Tools - **vs pip**: 10-100x faster, better resolver - **vs pip-tools**: Faster, simpler, better UX - **vs poetry**: Faster, less opinionated, lighter - **vs conda**: Faster, Python-focused ## Installation ### Quick Install ```bash # macOS/Linux curl -LsSf https://astral.sh/uv/install.sh | sh # Windows (PowerShell) powershell -c "irm https://astral.sh/uv/install.ps1 | iex" # Using pip (if you already have Python) pip install uv # Using Homebrew (macOS) brew install uv # Using cargo (if you have Rust) cargo install --git https://github.com/astral-sh/uv uv ``` ### Verify Installation ```bash uv --version # uv 0.x.x ``` ## Quick Start ### Create a New Project ```bash # Create new project with virtual environment uv init my-project cd my-project # Or create in current directory uv init . # Initialize creates: # - .python-version (Python version) # - pyproject.toml (project config) # - README.md # - .gitignore ``` ### Install Dependencies ```bash # Install packages (creates venv if needed) uv add requests pandas # Install dev dependencies uv add --dev pytest black ruff # Install from requirements.txt uv pip install -r requirements.txt # Install from pyproject.toml uv sync ``` ## Virtual Environment Management ### Pattern 1: Creating Virtual Environments ```bash # Create virtual environment with uv uv venv # Create with specific Python version uv venv --python 3.12 # Create with custom name uv venv my-env # Create with system site packages uv venv --system-site-packages # Specify location uv venv /path/to/venv ``` ### Pattern 2: Activating Virtual Environments ```bash # Linux/macOS source .venv/bin/activate # Windows (Command Prompt) .venv\Scripts\activate.bat # Windows (PowerShell) .venv\Scripts\Activate.ps1 # Or use uv run (no activation needed) uv run python script.py uv run pytest ``` ### Pattern 3: Using uv run ```bash # Run Python script (auto-activates venv) uv run python app.py # Run installed CLI tool uv run black . uv run pytest # Run with specific Python version uv run --python 3.11 python script.py # Pass arguments uv run python script.py --arg value ``` ## Package Management ### Pattern 4: Adding Dependencies ```bash # Add package (adds to pyproject.toml) uv add requests # Add with version constraint uv add "django>=4.0,<5.0" # Add multiple packages uv add numpy pandas matplotlib # Add dev dependency uv add --dev pytest pytest-cov # Add optional dependency group uv add --optional docs sphinx # Add from git uv add git+https://github.com/user/repo.git # Add from git with specific ref uv add git+https://github.com/user/repo.git@v1.0.0 # Add from local path uv add ./local-package # Add editable local package uv add -e ./local-package ``` ### Pattern 5: Removing Dependencies ```bash # Remove package uv remove requests # Remove dev dependency uv remove --dev pytest # Remove multiple packages uv remove numpy pandas matplotlib ``` ### Pattern 6: Upgrading Dependencies ```bash # Upgrade specific package uv add --upgrade requests # Upgrade all packages uv sync --upgrade # Upgrade package to latest uv add --upgrade requests # Show what would be upgraded uv tree --outdated ``` ### Pattern 7: Locking Dependencies ```bash # Generate uv.lock file uv lock # Update lock file uv lock --upgrade # Lock without installing uv lock --no-install # Lock specific package uv lock --upgrade-package requests ``` ## Python Version Management ### Pattern 8: Installing Python Versions ```bash # Install Python version uv python install 3.12 # Install multiple versions uv python install 3.11 3.12 3.13 # Install latest version uv python install # List installed versions uv python list # Find available versions uv python list --all-versions ``` ### Pattern 9: Setting Python Version ```bash # Set Python version for project uv python pin 3.12 # This creates/updates .python-version file # Use specific Python version for command uv --python 3.11 run python script.py # Create venv with specific version uv venv --python 3.12 ``` ## Project Configuration ### Pattern 10: pyproject.toml with uv ```toml [project] name = "my-project" version = "0.1.0" description = "My awesome project" readme = "README.md" requires-python = ">=3.8" dependencies = [ "requests>=2.31.0", "pydantic>=2.0.0", "click>=8.1.0", ] [project.optional-dependencies] dev = [ "pytest>=7.4.0", "pytest-cov>=4.1.0", "black>=23.0.0", "ruff>=0.1.0", "mypy>=1.5.0", ] docs = [ "sphinx>=7.0.0", "sphinx-rtd-theme>=1.3.0", ] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.uv] dev-dependencies = [ # Additional dev dependencies managed by uv ] [tool.uv.sources] # Custom package sources my-package = { git = "https://github.com/user/repo.git" } ``` ### Pattern 11: Using uv with Existing Projects ```bash # Migrate from requirements.txt uv add -r requirements.txt # Migrate from poetry # Already have pyproject.toml, just use: uv sync # Export to requirements.txt uv pip freeze > requirements.txt # Export with hashes uv pip freeze --require-hashes > requirements.txt ``` ## Advanced Workflows ### Pattern 12: Monorepo Support ```bash # Project structure # monorepo/ # packages/ # package-a/ # pyproject.toml # package-b/ # pyproject.toml # pyproject.toml (root) # Root pyproject.toml [tool.uv.workspace] members = ["packages/*"] # Install all workspace packages uv sync # Add workspace dependency uv add --path ./packages/package-a ``` ### Pattern 13: CI/CD Integration ```yaml # .github/workflows/test.yml name: Tests on: [push, pull_request] jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install uv uses: astral-sh/setup-uv@v2 with: enable-cache: true - name: Set up Python run: uv python install 3.12 - name: Install dependencies run: uv sync --all-extras --dev - name: Run tests run: uv run pytest - name: Run linting run: | uv run ruff check . uv run black --check . ``` ### Pattern 14: Docker Integration ```dockerfile # Dockerfile FROM python:3.12-slim # Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv # Set working directory WORKDIR /app # Copy dependency files COPY pyproject.toml uv.lock ./ # Install dependencies RUN uv sync --frozen --no-dev # Copy application code COPY . . # Run application CMD ["uv", "run", "python", "app.py"] ``` **Optimized multi-stage build:** ```dockerfile # Multi-stage Dockerfile FROM python:3.12-slim AS builder # Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv WORKDIR /app # Install dependencies to venv COPY pyproject.toml uv.lock ./ RUN uv sync --frozen --no-dev --no-editable # Runtime stage FROM python:3.12-slim WORKDIR /app # Copy venv from builder COPY --from=builder /app/.venv .venv COPY . . # Use venv ENV PATH="/app/.venv/bin:$PATH" CMD ["python", "app.py"] ``` ### Pattern 15: Lockfile Workflows ```bash # Create lockfile (uv.lock) uv lock # Install from lockfile (exact versions) uv sync --frozen # Update lockfile without installing uv lock --no-install # Upgrade specific package in lock uv lock --upgrade-package requests # Check if lockfile is up to date uv lock --check # Export lockfile to requirements.txt uv export --format requirements-txt > requirements.txt # Export with hashes for security uv export --format requirements-txt --hash > requirements.txt ``` ## Performance Optimization ### Pattern 16: Using Global Cache ```bash # UV automatically uses global cache at: # Linux: ~/.cache/uv # macOS: ~/Library/Caches/uv # Windows: %LOCALAPPDATA%\uv\cache # Clear cache uv cache clean # Check cache size uv cache dir ``` ### Pattern 17: Parallel Installation ```bash # UV installs packages in parallel by default # Control parallelism uv pip install --jobs 4 package1 package2 # No parallel (sequential) uv pip install --jobs 1 package ``` ### Pattern 18: Offline Mode ```bash # Install from cache only (no network) uv pip install --offline package # Sync from lockfile offline uv sync --frozen --offline ``` ## Comparison with Other Tools ### uv vs pip ```bash # pip python -m venv .venv source .venv/bin/activate pip install requests pandas numpy # ~30 seconds # uv uv venv uv add requests pandas numpy # ~2 seconds (10-15x faster) ``` ### uv vs poetry ```bash # poetry poetry init poetry add requests pandas poetry install # ~20 seconds # uv uv init uv add requests pandas uv sync # ~3 seconds (6-7x faster) ``` ### uv vs pip-tools ```bash # pip-tools pip-compile requirements.in pip-sync requirements.txt # ~15 seconds # uv uv lock uv sync --frozen # ~2 seconds (7-8x faster) ``` ## Common Workflows ### Pattern 19: Starting a New Project ```bash # Complete workflow uv init my-project cd my-project # Set Python version uv python pin 3.12 # Add dependencies uv add fastapi uvicorn pydantic # Add dev dependencies uv add --dev pytest black ruff mypy # Create structure mkdir -p src/my_project tests # Run tests uv run pytest # Format code uv run black . uv run ruff check . ``` ### Pattern 20: Maintaining Existing Project ```bash # Clone repository git clone https://github.com/user/project.git cd project # Install dependencies (creates venv automatically) uv sync # Install with dev dependencies uv sync --all-extras # Update dependencies uv lock --upgrade # Run application uv run python app.py # Run tests uv run pytest # Add new dependency uv add new-package # Commit updated files git add pyproject.toml uv.lock git commit -m "Add new-package dependency" ``` ## Tool Integration ### Pattern 21: Pre-commit Hooks ```yaml # .pre-commit-config.yaml repos: - repo: local hooks: - id: uv-lock name: uv lock entry: uv lock language: system pass_filenames: false - id: ruff name: ruff entry: uv run ruff check --fix language: system types: [python] - id: black name: black entry: uv run black language: system types: [python] ``` ### Pattern 22: VS Code Integration ```json // .vscode/settings.json { "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python", "python.terminal.activateEnvironment": true, "python.testing.pytestEnabled": true, "python.testing.pytestArgs": ["-v"], "python.linting.enabled": true, "python.formatting.provider": "black", "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true } } ``` ## Troubleshooting ### Common Issues ```bash # Issue: uv not found # Solution: Add to PATH or reinstall echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.bashrc # Issue: Wrong Python version # Solution: Pin version explicitly uv python pin 3.12 uv venv --python 3.12 # Issue: Dependency conflict # Solution: Check resolution uv lock --verbose # Issue: Cache issues # Solution: Clear cache uv cache clean # Issue: Lockfile out of sync # Solution: Regenerate uv lock --upgrade ``` ## Best Practices ### Project Setup 1. **Always use lockfiles** for reproducibility 2. **Pin Python version** with .python-version 3. **Separate dev dependencies** from production 4. **Use uv run** instead of activating venv 5. **Commit uv.lock** to version control 6. **Use --frozen in CI** for consistent builds 7. **Leverage global cache** for speed 8. **Use workspace** for monorepos 9. **Export requirements.txt** for compatibility 10. **Keep uv updated** for latest features ### Performance Tips ```bash # Use frozen installs in CI uv sync --frozen # Use offline mode when possible uv sync --offline # Parallel operations (automatic) # uv does this by default # Reuse cache across environments # uv shares cache globally # Use lockfiles to skip resolution uv sync --frozen # skips resolution ``` ## Migration Guide ### From pip + requirements.txt ```bash # Before python -m venv .venv source .venv/bin/activate pip install -r requirements.txt # After uv venv uv pip install -r requirements.txt # Or better: uv init uv add -r requirements.txt ``` ### From Poetry ```bash # Before poetry install poetry add requests # After uv sync uv add requests # Keep existing pyproject.toml # uv reads [project] and [tool.poetry] sections ``` ### From pip-tools ```bash # Before pip-compile requirements.in pip-sync requirements.txt # After uv lock uv sync --frozen ``` ## Command Reference ### Essential Commands ```bash # Project management uv init [PATH] # Initialize project uv add PACKAGE # Add dependency uv remove PACKAGE # Remove dependency uv sync # Install dependencies uv lock # Create/update lockfile # Virtual environments uv venv [PATH] # Create venv uv run COMMAND # Run in venv # Python management uv python install VERSION # Install Python uv python list # List installed Pythons uv python pin VERSION # Pin Python version # Package installation (pip-compatible) uv pip install PACKAGE # Install package uv pip uninstall PACKAGE # Uninstall package uv pip freeze # List installed uv pip list # List packages # Utility uv cache clean # Clear cache uv cache dir # Show cache location uv --version # Show version ``` ## Resources - **Official documentation**: https://docs.astral.sh/uv/ - **GitHub repository**: https://github.com/astral-sh/uv - **Astral blog**: https://astral.sh/blog - **Migration guides**: https://docs.astral.sh/uv/guides/ - **Comparison with other tools**: https://docs.astral.sh/uv/pip/compatibility/ ## Best Practices Summary 1. **Use uv for all new projects** - Start with `uv init` 2. **Commit lockfiles** - Ensure reproducible builds 3. **Pin Python versions** - Use .python-version 4. **Use uv run** - Avoid manual venv activation 5. **Leverage caching** - Let uv manage global cache 6. **Use --frozen in CI** - Exact reproduction 7. **Keep uv updated** - Fast-moving project 8. **Use workspaces** - For monorepo projects 9. **Export for compatibility** - Generate requirements.txt when needed 10. **Read the docs** - uv is feature-rich and evolving
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

memory-forensics

Master memory forensics techniques including memory acquisition,

security
⭐1
# Memory Forensics Comprehensive techniques for acquiring, analyzing, and extracting artifacts from memory dumps for incident response and malware analysis. ## Memory Acquisition ### Live Acquisition Tools #### Windows ```powershell # WinPmem (Recommended) winpmem_mini_x64.exe memory.raw # DumpIt DumpIt.exe # Belkasoft RAM Capturer # GUI-based, outputs raw format # Magnet RAM Capture # GUI-based, outputs raw format ``` #### Linux ```bash # LiME (Linux Memory Extractor) sudo insmod lime.ko "path=/tmp/memory.lime format=lime" # /dev/mem (limited, requires permissions) sudo dd if=/dev/mem of=memory.raw bs=1M # /proc/kcore (ELF format) sudo cp /proc/kcore memory.elf ``` #### macOS ```bash # osxpmem sudo ./osxpmem -o memory.raw # MacQuisition (commercial) ``` ### Virtual Machine Memory ```bash # VMware: .vmem file is raw memory cp vm.vmem memory.raw # VirtualBox: Use debug console vboxmanage debugvm "VMName" dumpvmcore --filename memory.elf # QEMU virsh dump <domain> memory.raw --memory-only # Hyper-V # Checkpoint contains memory state ``` ## Volatility 3 Framework ### Installation and Setup ```bash # Install Volatility 3 pip install volatility3 # Install symbol tables (Windows) # Download from https://downloads.volatilityfoundation.org/volatility3/symbols/ # Basic usage vol -f memory.raw <plugin> # With symbol path vol -f memory.raw -s /path/to/symbols windows.pslist ``` ### Essential Plugins #### Process Analysis ```bash # List processes vol -f memory.raw windows.pslist # Process tree (parent-child relationships) vol -f memory.raw windows.pstree # Hidden process detection vol -f memory.raw windows.psscan # Process memory dumps vol -f memory.raw windows.memmap --pid <PID> --dump # Process environment variables vol -f memory.raw windows.envars --pid <PID> # Command line arguments vol -f memory.raw windows.cmdline ``` #### Network Analysis ```bash # Network connections vol -f memory.raw windows.netscan # Network connection state vol -f memory.raw windows.netstat ``` #### DLL and Module Analysis ```bash # Loaded DLLs per process vol -f memory.raw windows.dlllist --pid <PID> # Find hidden/injected DLLs vol -f memory.raw windows.ldrmodules # Kernel modules vol -f memory.raw windows.modules # Module dumps vol -f memory.raw windows.moddump --pid <PID> ``` #### Memory Injection Detection ```bash # Detect code injection vol -f memory.raw windows.malfind # VAD (Virtual Address Descriptor) analysis vol -f memory.raw windows.vadinfo --pid <PID> # Dump suspicious memory regions vol -f memory.raw windows.vadyarascan --yara-rules rules.yar ``` #### Registry Analysis ```bash # List registry hives vol -f memory.raw windows.registry.hivelist # Print registry key vol -f memory.raw windows.registry.printkey --key "Software\Microsoft\Windows\CurrentVersion\Run" # Dump registry hive vol -f memory.raw windows.registry.hivescan --dump ``` #### File System Artifacts ```bash # Scan for file objects vol -f memory.raw windows.filescan # Dump files from memory vol -f memory.raw windows.dumpfiles --pid <PID> # MFT analysis vol -f memory.raw windows.mftscan ``` ### Linux Analysis ```bash # Process listing vol -f memory.raw linux.pslist # Process tree vol -f memory.raw linux.pstree # Bash history vol -f memory.raw linux.bash # Network connections vol -f memory.raw linux.sockstat # Loaded kernel modules vol -f memory.raw linux.lsmod # Mount points vol -f memory.raw linux.mount # Environment variables vol -f memory.raw linux.envars ``` ### macOS Analysis ```bash # Process listing vol -f memory.raw mac.pslist # Process tree vol -f memory.raw mac.pstree # Network connections vol -f memory.raw mac.netstat # Kernel extensions vol -f memory.raw mac.lsmod ``` ## Analysis Workflows ### Malware Analysis Workflow ```bash # 1. Initial process survey vol -f memory.raw windows.pstree > processes.txt vol -f memory.raw windows.pslist > pslist.txt # 2. Network connections vol -f memory.raw windows.netscan > network.txt # 3. Detect injection vol -f memory.raw windows.malfind > malfind.txt # 4. Analyze suspicious processes vol -f memory.raw windows.dlllist --pid <PID> vol -f memory.raw windows.handles --pid <PID> # 5. Dump suspicious executables vol -f memory.raw windows.pslist --pid <PID> --dump # 6. Extract strings from dumps strings -a pid.<PID>.exe > strings.txt # 7. YARA scanning vol -f memory.raw windows.yarascan --yara-rules malware.yar ``` ### Incident Response Workflow ```bash # 1. Timeline of events vol -f memory.raw windows.timeliner > timeline.csv # 2. User activity vol -f memory.raw windows.cmdline vol -f memory.raw windows.consoles # 3. Persistence mechanisms vol -f memory.raw windows.registry.printkey \ --key "Software\Microsoft\Windows\CurrentVersion\Run" # 4. Services vol -f memory.raw windows.svcscan # 5. Scheduled tasks vol -f memory.raw windows.scheduled_tasks # 6. Recent files vol -f memory.raw windows.filescan | grep -i "recent" ``` ## Data Structures ### Windows Process Structures ```c // EPROCESS (Executive Process) typedef struct _EPROCESS { KPROCESS Pcb; // Kernel process block EX_PUSH_LOCK ProcessLock; LARGE_INTEGER CreateTime; LARGE_INTEGER ExitTime; // ... LIST_ENTRY ActiveProcessLinks; // Doubly-linked list ULONG_PTR UniqueProcessId; // PID // ... PEB* Peb; // Process Environment Block // ... } EPROCESS; // PEB (Process Environment Block) typedef struct _PEB { BOOLEAN InheritedAddressSpace; BOOLEAN ReadImageFileExecOptions; BOOLEAN BeingDebugged; // Anti-debug check // ... PVOID ImageBaseAddress; // Base address of executable PPEB_LDR_DATA Ldr; // Loader data (DLL list) PRTL_USER_PROCESS_PARAMETERS ProcessParameters; // ... } PEB; ``` ### VAD (Virtual Address Descriptor) ```c typedef struct _MMVAD { MMVAD_SHORT Core; union { ULONG LongFlags; MMVAD_FLAGS VadFlags; } u; // ... PVOID FirstPrototypePte; PVOID LastContiguousPte; // ... PFILE_OBJECT FileObject; } MMVAD; // Memory protection flags #define PAGE_EXECUTE 0x10 #define PAGE_EXECUTE_READ 0x20 #define PAGE_EXECUTE_READWRITE 0x40 #define PAGE_EXECUTE_WRITECOPY 0x80 ``` ## Detection Patterns ### Process Injection Indicators ```python # Malfind indicators # - PAGE_EXECUTE_READWRITE protection (suspicious) # - MZ header in non-image VAD region # - Shellcode patterns at allocation start # Common injection techniques # 1. Classic DLL Injection # - VirtualAllocEx + WriteProcessMemory + CreateRemoteThread # 2. Process Hollowing # - CreateProcess (SUSPENDED) + NtUnmapViewOfSection + WriteProcessMemory # 3. APC Injection # - QueueUserAPC targeting alertable threads # 4. Thread Execution Hijacking # - SuspendThread + SetThreadContext + ResumeThread ``` ### Rootkit Detection ```bash # Compare process lists vol -f memory.raw windows.pslist > pslist.txt vol -f memory.raw windows.psscan > psscan.txt diff pslist.txt psscan.txt # Hidden processes # Check for DKOM (Direct Kernel Object Manipulation) vol -f memory.raw windows.callbacks # Detect hooked functions vol -f memory.raw windows.ssdt # System Service Descriptor Table # Driver analysis vol -f memory.raw windows.driverscan vol -f memory.raw windows.driverirp ``` ### Credential Extraction ```bash # Dump hashes (requires hivelist first) vol -f memory.raw windows.hashdump # LSA secrets vol -f memory.raw windows.lsadump # Cached domain credentials vol -f memory.raw windows.cachedump # Mimikatz-style extraction # Requires specific plugins/tools ``` ## YARA Integration ### Writing Memory YARA Rules ```yara rule Suspicious_Injection { meta: description = "Detects common injection shellcode" strings: // Common shellcode patterns $mz = { 4D 5A } $shellcode1 = { 55 8B EC 83 EC } // Function prologue $api_hash = { 68 ?? ?? ?? ?? 68 ?? ?? ?? ?? E8 } // Push hash, call condition: $mz at 0 or any of ($shellcode*) } rule Cobalt_Strike_Beacon { meta: description = "Detects Cobalt Strike beacon in memory" strings: $config = { 00 01 00 01 00 02 } $sleep = "sleeptime" $beacon = "%s (admin)" wide condition: 2 of them } ``` ### Scanning Memory ```bash # Scan all process memory vol -f memory.raw windows.yarascan --yara-rules rules.yar # Scan specific process vol -f memory.raw windows.yarascan --yara-rules rules.yar --pid 1234 # Scan kernel memory vol -f memory.raw windows.yarascan --yara-rules rules.yar --kernel ``` ## String Analysis ### Extracting Strings ```bash # Basic string extraction strings -a memory.raw > all_strings.txt # Unicode strings strings -el memory.raw >> all_strings.txt # Targeted extraction from process dump vol -f memory.raw windows.memmap --pid 1234 --dump strings -a pid.1234.dmp > process_strings.txt # Pattern matching grep -E "(https?://|[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})" all_strings.txt ``` ### FLOSS for Obfuscated Strings ```bash # FLOSS extracts obfuscated strings floss malware.exe > floss_output.txt # From memory dump floss pid.1234.dmp ``` ## Best Practices ### Acquisition Best Practices 1. **Minimize footprint**: Use lightweight acquisition tools 2. **Document everything**: Record time, tool, and hash of capture 3. **Verify integrity**: Hash memory dump immediately after capture 4. **Chain of custody**: Maintain proper forensic handling ### Analysis Best Practices 1. **Start broad**: Get overview before deep diving 2. **Cross-reference**: Use multiple plugins for same data 3. **Timeline correlation**: Correlate memory findings with disk/network 4. **Document findings**: Keep detailed notes and screenshots 5. **Validate results**: Verify findings through multiple methods ### Common Pitfalls - **Stale data**: Memory is volatile, analyze promptly - **Incomplete dumps**: Verify dump size matches expected RAM - **Symbol issues**: Ensure correct symbol files for OS version - **Smear**: Memory may change during acquisition - **Encryption**: Some data may be encrypted in memory
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

protocol-reverse-engineering

Master network protocol reverse engineering including packet

security
⭐1
# Protocol Reverse Engineering Comprehensive techniques for capturing, analyzing, and documenting network protocols for security research, interoperability, and debugging. ## Traffic Capture ### Wireshark Capture ```bash # Capture on specific interface wireshark -i eth0 -k # Capture with filter wireshark -i eth0 -k -f "port 443" # Capture to file tshark -i eth0 -w capture.pcap # Ring buffer capture (rotate files) tshark -i eth0 -b filesize:100000 -b files:10 -w capture.pcap ``` ### tcpdump Capture ```bash # Basic capture tcpdump -i eth0 -w capture.pcap # With filter tcpdump -i eth0 port 8080 -w capture.pcap # Capture specific bytes tcpdump -i eth0 -s 0 -w capture.pcap # Full packet # Real-time display tcpdump -i eth0 -X port 80 ``` ### Man-in-the-Middle Capture ```bash # mitmproxy for HTTP/HTTPS mitmproxy --mode transparent -p 8080 # SSL/TLS interception mitmproxy --mode transparent --ssl-insecure # Dump to file mitmdump -w traffic.mitm # Burp Suite # Configure browser proxy to 127.0.0.1:8080 ``` ## Protocol Analysis ### Wireshark Analysis ``` # Display filters tcp.port == 8080 http.request.method == "POST" ip.addr == 192.168.1.1 tcp.flags.syn == 1 && tcp.flags.ack == 0 frame contains "password" # Following streams Right-click > Follow > TCP Stream Right-click > Follow > HTTP Stream # Export objects File > Export Objects > HTTP # Decryption Edit > Preferences > Protocols > TLS - (Pre)-Master-Secret log filename - RSA keys list ``` ### tshark Analysis ```bash # Extract specific fields tshark -r capture.pcap -T fields -e ip.src -e ip.dst -e tcp.port # Statistics tshark -r capture.pcap -q -z conv,tcp tshark -r capture.pcap -q -z endpoints,ip # Filter and extract tshark -r capture.pcap -Y "http" -T json > http_traffic.json # Protocol hierarchy tshark -r capture.pcap -q -z io,phs ``` ### Scapy for Custom Analysis ```python from scapy.all import * # Read pcap packets = rdpcap("capture.pcap") # Analyze packets for pkt in packets: if pkt.haslayer(TCP): print(f"Src: {pkt[IP].src}:{pkt[TCP].sport}") print(f"Dst: {pkt[IP].dst}:{pkt[TCP].dport}") if pkt.haslayer(Raw): print(f"Data: {pkt[Raw].load[:50]}") # Filter packets http_packets = [p for p in packets if p.haslayer(TCP) and (p[TCP].sport == 80 or p[TCP].dport == 80)] # Create custom packets pkt = IP(dst="target")/TCP(dport=80)/Raw(load="GET / HTTP/1.1\r\n") send(pkt) ``` ## Protocol Identification ### Common Protocol Signatures ``` HTTP - "HTTP/1." or "GET " or "POST " at start TLS/SSL - 0x16 0x03 (record layer) DNS - UDP port 53, specific header format SMB - 0xFF 0x53 0x4D 0x42 ("SMB" signature) SSH - "SSH-2.0" banner FTP - "220 " response, "USER " command SMTP - "220 " banner, "EHLO" command MySQL - 0x00 length prefix, protocol version PostgreSQL - 0x00 0x00 0x00 startup length Redis - "*" RESP array prefix MongoDB - BSON documents with specific header ``` ### Protocol Header Patterns ``` +--------+--------+--------+--------+ | Magic number / Signature | +--------+--------+--------+--------+ | Version | Flags | +--------+--------+--------+--------+ | Length | Message Type | +--------+--------+--------+--------+ | Sequence Number / Session ID | +--------+--------+--------+--------+ | Payload... | +--------+--------+--------+--------+ ``` ## Binary Protocol Analysis ### Structure Identification ```python # Common patterns in binary protocols # Length-prefixed message struct Message { uint32_t length; # Total message length uint16_t msg_type; # Message type identifier uint8_t flags; # Flags/options uint8_t reserved; # Padding/alignment uint8_t payload[]; # Variable-length payload }; # Type-Length-Value (TLV) struct TLV { uint8_t type; # Field type uint16_t length; # Field length uint8_t value[]; # Field data }; # Fixed header + variable payload struct Packet { uint8_t magic[4]; # "ABCD" signature uint32_t version; uint32_t payload_len; uint32_t checksum; # CRC32 or similar uint8_t payload[]; }; ``` ### Python Protocol Parser ```python import struct from dataclasses import dataclass @dataclass class MessageHeader: magic: bytes version: int msg_type: int length: int @classmethod def from_bytes(cls, data: bytes): magic, version, msg_type, length = struct.unpack( ">4sHHI", data[:12] ) return cls(magic, version, msg_type, length) def parse_messages(data: bytes): offset = 0 messages = [] while offset < len(data): header = MessageHeader.from_bytes(data[offset:]) payload = data[offset+12:offset+12+header.length] messages.append((header, payload)) offset += 12 + header.length return messages # Parse TLV structure def parse_tlv(data: bytes): fields = [] offset = 0 while offset < len(data): field_type = data[offset] length = struct.unpack(">H", data[offset+1:offset+3])[0] value = data[offset+3:offset+3+length] fields.append((field_type, value)) offset += 3 + length return fields ``` ### Hex Dump Analysis ```python def hexdump(data: bytes, width: int = 16): """Format binary data as hex dump.""" lines = [] for i in range(0, len(data), width): chunk = data[i:i+width] hex_part = ' '.join(f'{b:02x}' for b in chunk) ascii_part = ''.join( chr(b) if 32 <= b < 127 else '.' for b in chunk ) lines.append(f'{i:08x} {hex_part:<{width*3}} {ascii_part}') return '\n'.join(lines) # Example output: # 00000000 48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d HTTP/1.1 200 OK. # 00000010 0a 43 6f 6e 74 65 6e 74 2d 54 79 70 65 3a 20 74 .Content-Type: t ``` ## Encryption Analysis ### Identifying Encryption ```python # Entropy analysis - high entropy suggests encryption/compression import math from collections import Counter def entropy(data: bytes) -> float: if not data: return 0.0 counter = Counter(data) probs = [count / len(data) for count in counter.values()] return -sum(p * math.log2(p) for p in probs) # Entropy thresholds: # < 6.0: Likely plaintext or structured data # 6.0-7.5: Possibly compressed # > 7.5: Likely encrypted or random # Common encryption indicators # - High, uniform entropy # - No obvious structure or patterns # - Length often multiple of block size (16 for AES) # - Possible IV at start (16 bytes for AES-CBC) ``` ### TLS Analysis ```bash # Extract TLS metadata tshark -r capture.pcap -Y "ssl.handshake" \ -T fields -e ip.src -e ssl.handshake.ciphersuite # JA3 fingerprinting (client) tshark -r capture.pcap -Y "ssl.handshake.type == 1" \ -T fields -e ssl.handshake.ja3 # JA3S fingerprinting (server) tshark -r capture.pcap -Y "ssl.handshake.type == 2" \ -T fields -e ssl.handshake.ja3s # Certificate extraction tshark -r capture.pcap -Y "ssl.handshake.certificate" \ -T fields -e x509sat.printableString ``` ### Decryption Approaches ```bash # Pre-master secret log (browser) export SSLKEYLOGFILE=/tmp/keys.log # Configure Wireshark # Edit > Preferences > Protocols > TLS # (Pre)-Master-Secret log filename: /tmp/keys.log # Decrypt with private key (if available) # Only works for RSA key exchange # Edit > Preferences > Protocols > TLS > RSA keys list ``` ## Custom Protocol Documentation ### Protocol Specification Template ```markdown # Protocol Name Specification ## Overview Brief description of protocol purpose and design. ## Transport - Layer: TCP/UDP - Port: XXXX - Encryption: TLS 1.2+ ## Message Format ### Header (12 bytes) | Offset | Size | Field | Description | | ------ | ---- | ------- | ----------------------- | | 0 | 4 | Magic | 0x50524F54 ("PROT") | | 4 | 2 | Version | Protocol version (1) | | 6 | 2 | Type | Message type identifier | | 8 | 4 | Length | Payload length in bytes | ### Message Types | Type | Name | Description | | ---- | --------- | ---------------------- | | 0x01 | HELLO | Connection initiation | | 0x02 | HELLO_ACK | Connection accepted | | 0x03 | DATA | Application data | | 0x04 | CLOSE | Connection termination | ### Type 0x01: HELLO | Offset | Size | Field | Description | | ------ | ---- | ---------- | ------------------------ | | 0 | 4 | ClientID | Unique client identifier | | 4 | 2 | Flags | Connection flags | | 6 | var | Extensions | TLV-encoded extensions | ## State Machine ``` [INIT] --HELLO--> [WAIT_ACK] --HELLO_ACK--> [CONNECTED] | DATA/DATA | [CLOSED] <--CLOSE--+ ``` ## Examples ### Connection Establishment ``` Client -> Server: HELLO (ClientID=0x12345678) Server -> Client: HELLO_ACK (Status=OK) Client -> Server: DATA (payload) ``` ``` ### Wireshark Dissector (Lua) ```lua -- custom_protocol.lua local proto = Proto("custom", "Custom Protocol") -- Define fields local f_magic = ProtoField.string("custom.magic", "Magic") local f_version = ProtoField.uint16("custom.version", "Version") local f_type = ProtoField.uint16("custom.type", "Type") local f_length = ProtoField.uint32("custom.length", "Length") local f_payload = ProtoField.bytes("custom.payload", "Payload") proto.fields = { f_magic, f_version, f_type, f_length, f_payload } -- Message type names local msg_types = { [0x01] = "HELLO", [0x02] = "HELLO_ACK", [0x03] = "DATA", [0x04] = "CLOSE" } function proto.dissector(buffer, pinfo, tree) pinfo.cols.protocol = "CUSTOM" local subtree = tree:add(proto, buffer()) -- Parse header subtree:add(f_magic, buffer(0, 4)) subtree:add(f_version, buffer(4, 2)) local msg_type = buffer(6, 2):uint() subtree:add(f_type, buffer(6, 2)):append_text( " (" .. (msg_types[msg_type] or "Unknown") .. ")" ) local length = buffer(8, 4):uint() subtree:add(f_length, buffer(8, 4)) if length > 0 then subtree:add(f_payload, buffer(12, length)) end end -- Register for TCP port local tcp_table = DissectorTable.get("tcp.port") tcp_table:add(8888, proto) ``` ## Active Testing ### Fuzzing with Boofuzz ```python from boofuzz import * def main(): session = Session( target=Target( connection=TCPSocketConnection("target", 8888) ) ) # Define protocol structure s_initialize("HELLO") s_static(b"\x50\x52\x4f\x54") # Magic s_word(1, name="version") # Version s_word(0x01, name="type") # Type (HELLO) s_size("payload", length=4) # Length field s_block_start("payload") s_dword(0x12345678, name="client_id") s_word(0, name="flags") s_block_end() session.connect(s_get("HELLO")) session.fuzz() if __name__ == "__main__": main() ``` ### Replay and Modification ```python from scapy.all import * # Replay captured traffic packets = rdpcap("capture.pcap") for pkt in packets: if pkt.haslayer(TCP) and pkt[TCP].dport == 8888: send(pkt) # Modify and replay for pkt in packets: if pkt.haslayer(Raw): # Modify payload original = pkt[Raw].load modified = original.replace(b"client", b"CLIENT") pkt[Raw].load = modified # Recalculate checksums del pkt[IP].chksum del pkt[TCP].chksum send(pkt) ``` ## Best Practices ### Analysis Workflow 1. **Capture traffic**: Multiple sessions, different scenarios 2. **Identify boundaries**: Message start/end markers 3. **Map structure**: Fixed header, variable payload 4. **Identify fields**: Compare multiple samples 5. **Document format**: Create specification 6. **Validate understanding**: Implement parser/generator 7. **Test edge cases**: Fuzzing, boundary conditions ### Common Patterns to Look For - Magic numbers/signatures at message start - Version fields for compatibility - Length fields (often before variable data) - Type/opcode fields for message identification - Sequence numbers for ordering - Checksums/CRCs for integrity - Timestamps for timing - Session/connection identifiers
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

attack-tree-construction

Build comprehensive attack trees to visualize threat paths. Use

security
⭐1
# Attack Tree Construction Systematic attack path visualization and analysis. ## When to Use This Skill - Visualizing complex attack scenarios - Identifying defense gaps and priorities - Communicating risks to stakeholders - Planning defensive investments - Penetration test planning - Security architecture review ## Core Concepts ### 1. Attack Tree Structure ``` [Root Goal] | ┌────────────┴────────────┐ │ │ [Sub-goal 1] [Sub-goal 2] (OR node) (AND node) │ │ ┌─────┴─────┐ ┌─────┴─────┐ │ │ │ │ [Attack] [Attack] [Attack] [Attack] (leaf) (leaf) (leaf) (leaf) ``` ### 2. Node Types | Type | Symbol | Description | | -------- | --------- | ----------------------- | | **OR** | Oval | Any child achieves goal | | **AND** | Rectangle | All children required | | **Leaf** | Box | Atomic attack step | ### 3. Attack Attributes | Attribute | Description | Values | | ------------- | ----------------------- | ------------------ | | **Cost** | Resources needed | $, $$, $$$ | | **Time** | Duration to execute | Hours, Days, Weeks | | **Skill** | Expertise required | Low, Medium, High | | **Detection** | Likelihood of detection | Low, Medium, High | ## Templates ### Template 1: Attack Tree Data Model ```python from dataclasses import dataclass, field from enum import Enum from typing import List, Dict, Optional, Union import json class NodeType(Enum): OR = "or" AND = "and" LEAF = "leaf" class Difficulty(Enum): TRIVIAL = 1 LOW = 2 MEDIUM = 3 HIGH = 4 EXPERT = 5 class Cost(Enum): FREE = 0 LOW = 1 MEDIUM = 2 HIGH = 3 VERY_HIGH = 4 class DetectionRisk(Enum): NONE = 0 LOW = 1 MEDIUM = 2 HIGH = 3 CERTAIN = 4 @dataclass class AttackAttributes: difficulty: Difficulty = Difficulty.MEDIUM cost: Cost = Cost.MEDIUM detection_risk: DetectionRisk = DetectionRisk.MEDIUM time_hours: float = 8.0 requires_insider: bool = False requires_physical: bool = False @dataclass class AttackNode: id: str name: str description: str node_type: NodeType attributes: AttackAttributes = field(default_factory=AttackAttributes) children: List['AttackNode'] = field(default_factory=list) mitigations: List[str] = field(default_factory=list) cve_refs: List[str] = field(default_factory=list) def add_child(self, child: 'AttackNode') -> None: self.children.append(child) def calculate_path_difficulty(self) -> float: """Calculate aggregate difficulty for this path.""" if self.node_type == NodeType.LEAF: return self.attributes.difficulty.value if not self.children: return 0 child_difficulties = [c.calculate_path_difficulty() for c in self.children] if self.node_type == NodeType.OR: return min(child_difficulties) else: # AND return max(child_difficulties) def calculate_path_cost(self) -> float: """Calculate aggregate cost for this path.""" if self.node_type == NodeType.LEAF: return self.attributes.cost.value if not self.children: return 0 child_costs = [c.calculate_path_cost() for c in self.children] if self.node_type == NodeType.OR: return min(child_costs) else: # AND return sum(child_costs) def to_dict(self) -> Dict: """Convert to dictionary for serialization.""" return { "id": self.id, "name": self.name, "description": self.description, "type": self.node_type.value, "attributes": { "difficulty": self.attributes.difficulty.name, "cost": self.attributes.cost.name, "detection_risk": self.attributes.detection_risk.name, "time_hours": self.attributes.time_hours, }, "mitigations": self.mitigations, "children": [c.to_dict() for c in self.children] } @dataclass class AttackTree: name: str description: str root: AttackNode version: str = "1.0" def find_easiest_path(self) -> List[AttackNode]: """Find the path with lowest difficulty.""" return self._find_path(self.root, minimize="difficulty") def find_cheapest_path(self) -> List[AttackNode]: """Find the path with lowest cost.""" return self._find_path(self.root, minimize="cost") def find_stealthiest_path(self) -> List[AttackNode]: """Find the path with lowest detection risk.""" return self._find_path(self.root, minimize="detection") def _find_path( self, node: AttackNode, minimize: str ) -> List[AttackNode]: """Recursive path finding.""" if node.node_type == NodeType.LEAF: return [node] if not node.children: return [node] if node.node_type == NodeType.OR: # Pick the best child path best_path = None best_score = float('inf') for child in node.children: child_path = self._find_path(child, minimize) score = self._path_score(child_path, minimize) if score < best_score: best_score = score best_path = child_path return [node] + (best_path or []) else: # AND # Must traverse all children path = [node] for child in node.children: path.extend(self._find_path(child, minimize)) return path def _path_score(self, path: List[AttackNode], metric: str) -> float: """Calculate score for a path.""" if metric == "difficulty": return sum(n.attributes.difficulty.value for n in path if n.node_type == NodeType.LEAF) elif metric == "cost": return sum(n.attributes.cost.value for n in path if n.node_type == NodeType.LEAF) elif metric == "detection": return sum(n.attributes.detection_risk.value for n in path if n.node_type == NodeType.LEAF) return 0 def get_all_leaf_attacks(self) -> List[AttackNode]: """Get all leaf attack nodes.""" leaves = [] self._collect_leaves(self.root, leaves) return leaves def _collect_leaves(self, node: AttackNode, leaves: List[AttackNode]) -> None: if node.node_type == NodeType.LEAF: leaves.append(node) for child in node.children: self._collect_leaves(child, leaves) def get_unmitigated_attacks(self) -> List[AttackNode]: """Find attacks without mitigations.""" return [n for n in self.get_all_leaf_attacks() if not n.mitigations] def export_json(self) -> str: """Export tree to JSON.""" return json.dumps({ "name": self.name, "description": self.description, "version": self.version, "root": self.root.to_dict() }, indent=2) ``` ### Template 2: Attack Tree Builder ```python class AttackTreeBuilder: """Fluent builder for attack trees.""" def __init__(self, name: str, description: str): self.name = name self.description = description self._node_stack: List[AttackNode] = [] self._root: Optional[AttackNode] = None def goal(self, id: str, name: str, description: str = "") -> 'AttackTreeBuilder': """Set the root goal (OR node by default).""" self._root = AttackNode( id=id, name=name, description=description, node_type=NodeType.OR ) self._node_stack = [self._root] return self def or_node(self, id: str, name: str, description: str = "") -> 'AttackTreeBuilder': """Add an OR sub-goal.""" node = AttackNode( id=id, name=name, description=description, node_type=NodeType.OR ) self._current().add_child(node) self._node_stack.append(node) return self def and_node(self, id: str, name: str, description: str = "") -> 'AttackTreeBuilder': """Add an AND sub-goal (all children required).""" node = AttackNode( id=id, name=name, description=description, node_type=NodeType.AND ) self._current().add_child(node) self._node_stack.append(node) return self def attack( self, id: str, name: str, description: str = "", difficulty: Difficulty = Difficulty.MEDIUM, cost: Cost = Cost.MEDIUM, detection: DetectionRisk = DetectionRisk.MEDIUM, time_hours: float = 8.0, mitigations: List[str] = None ) -> 'AttackTreeBuilder': """Add a leaf attack node.""" node = AttackNode( id=id, name=name, description=description, node_type=NodeType.LEAF, attributes=AttackAttributes( difficulty=difficulty, cost=cost, detection_risk=detection, time_hours=time_hours ), mitigations=mitigations or [] ) self._current().add_child(node) return self def end(self) -> 'AttackTreeBuilder': """Close current node, return to parent.""" if len(self._node_stack) > 1: self._node_stack.pop() return self def build(self) -> AttackTree: """Build the attack tree.""" if not self._root: raise ValueError("No root goal defined") return AttackTree( name=self.name, description=self.description, root=self._root ) def _current(self) -> AttackNode: if not self._node_stack: raise ValueError("No current node") return self._node_stack[-1] # Example usage def build_account_takeover_tree() -> AttackTree: """Build attack tree for account takeover scenario.""" return ( AttackTreeBuilder("Account Takeover", "Gain unauthorized access to user account") .goal("G1", "Take Over User Account") .or_node("S1", "Steal Credentials") .attack( "A1", "Phishing Attack", difficulty=Difficulty.LOW, cost=Cost.LOW, detection=DetectionRisk.MEDIUM, mitigations=["Security awareness training", "Email filtering"] ) .attack( "A2", "Credential Stuffing", difficulty=Difficulty.TRIVIAL, cost=Cost.LOW, detection=DetectionRisk.HIGH, mitigations=["Rate limiting", "MFA", "Password breach monitoring"] ) .attack( "A3", "Keylogger Malware", difficulty=Difficulty.MEDIUM, cost=Cost.MEDIUM, detection=DetectionRisk.MEDIUM, mitigations=["Endpoint protection", "MFA"] ) .end() .or_node("S2", "Bypass Authentication") .attack( "A4", "Session Hijacking", difficulty=Difficulty.MEDIUM, cost=Cost.LOW, detection=DetectionRisk.LOW, mitigations=["Secure session management", "HTTPS only"] ) .attack( "A5", "Authentication Bypass Vulnerability", difficulty=Difficulty.HIGH, cost=Cost.LOW, detection=DetectionRisk.LOW, mitigations=["Security testing", "Code review", "WAF"] ) .end() .or_node("S3", "Social Engineering") .and_node("S3.1", "Account Recovery Attack") .attack( "A6", "Gather Personal Information", difficulty=Difficulty.LOW, cost=Cost.FREE, detection=DetectionRisk.NONE ) .attack( "A7", "Call Support Desk", difficulty=Difficulty.MEDIUM, cost=Cost.FREE, detection=DetectionRisk.MEDIUM, mitigations=["Support verification procedures", "Security questions"] ) .end() .end() .build() ) ``` ### Template 3: Mermaid Diagram Generator ```python class MermaidExporter: """Export attack trees to Mermaid diagram format.""" def __init__(self, tree: AttackTree): self.tree = tree self._lines: List[str] = [] self._node_count = 0 def export(self) -> str: """Export tree to Mermaid flowchart.""" self._lines = ["flowchart TD"] self._export_node(self.tree.root, None) return "\n".join(self._lines) def _export_node(self, node: AttackNode, parent_id: Optional[str]) -> str: """Recursively export nodes.""" node_id = f"N{self._node_count}" self._node_count += 1 # Node shape based on type if node.node_type == NodeType.OR: shape = f"{node_id}(({node.name}))" elif node.node_type == NodeType.AND: shape = f"{node_id}[{node.name}]" else: # LEAF # Color based on difficulty style = self._get_leaf_style(node) shape = f"{node_id}[/{node.name}/]" self._lines.append(f" style {node_id} {style}") self._lines.append(f" {shape}") if parent_id: connector = "-->" if node.node_type != NodeType.AND else "==>" self._lines.append(f" {parent_id} {connector} {node_id}") for child in node.children: self._export_node(child, node_id) return node_id def _get_leaf_style(self, node: AttackNode) -> str: """Get style based on attack attributes.""" colors = { Difficulty.TRIVIAL: "fill:#ff6b6b", # Red - easy attack Difficulty.LOW: "fill:#ffa06b", Difficulty.MEDIUM: "fill:#ffd93d", Difficulty.HIGH: "fill:#6bcb77", Difficulty.EXPERT: "fill:#4d96ff", # Blue - hard attack } color = colors.get(node.attributes.difficulty, "fill:#gray") return color class PlantUMLExporter: """Export attack trees to PlantUML format.""" def __init__(self, tree: AttackTree): self.tree = tree def export(self) -> str: """Export tree to PlantUML.""" lines = [ "@startmindmap", f"* {self.tree.name}", ] self._export_node(self.tree.root, lines, 1) lines.append("@endmindmap") return "\n".join(lines) def _export_node(self, node: AttackNode, lines: List[str], depth: int) -> None: """Recursively export nodes.""" prefix = "*" * (depth + 1) if node.node_type == NodeType.OR: marker = "[OR]" elif node.node_type == NodeType.AND: marker = "[AND]" else: diff = node.attributes.difficulty.name marker = f"<<{diff}>>" lines.append(f"{prefix} {marker} {node.name}") for child in node.children: self._export_node(child, lines, depth + 1) ``` ### Template 4: Attack Path Analysis ```python from typing import Set, Tuple class AttackPathAnalyzer: """Analyze attack paths and coverage.""" def __init__(self, tree: AttackTree): self.tree = tree def get_all_paths(self) -> List[List[AttackNode]]: """Get all possible attack paths.""" paths = [] self._collect_paths(self.tree.root, [], paths) return paths def _collect_paths( self, node: AttackNode, current_path: List[AttackNode], all_paths: List[List[AttackNode]] ) -> None: """Recursively collect all paths.""" current_path = current_path + [node] if node.node_type == NodeType.LEAF: all_paths.append(current_path) return if not node.children: all_paths.append(current_path) return if node.node_type == NodeType.OR: # Each child is a separate path for child in node.children: self._collect_paths(child, current_path, all_paths) else: # AND # Must combine all children child_paths = [] for child in node.children: child_sub_paths = [] self._collect_paths(child, [], child_sub_paths) child_paths.append(child_sub_paths) # Combine paths from all AND children combined = self._combine_and_paths(child_paths) for combo in combined: all_paths.append(current_path + combo) def _combine_and_paths( self, child_paths: List[List[List[AttackNode]]] ) -> List[List[AttackNode]]: """Combine paths from AND node children.""" if not child_paths: return [[]] if len(child_paths) == 1: return [path for paths in child_paths for path in paths] # Cartesian product of all child path combinations result = [[]] for paths in child_paths: new_result = [] for existing in result: for path in paths: new_result.append(existing + path) result = new_result return result def calculate_path_metrics(self, path: List[AttackNode]) -> Dict: """Calculate metrics for a specific path.""" leaves = [n for n in path if n.node_type == NodeType.LEAF] total_difficulty = sum(n.attributes.difficulty.value for n in leaves) total_cost = sum(n.attributes.cost.value for n in leaves) total_time = sum(n.attributes.time_hours for n in leaves) max_detection = max((n.attributes.detection_risk.value for n in leaves), default=0) return { "steps": len(leaves), "total_difficulty": total_difficulty, "avg_difficulty": total_difficulty / len(leaves) if leaves else 0, "total_cost": total_cost, "total_time_hours": total_time, "max_detection_risk": max_detection, "requires_insider": any(n.attributes.requires_insider for n in leaves), "requires_physical": any(n.attributes.requires_physical for n in leaves), } def identify_critical_nodes(self) -> List[Tuple[AttackNode, int]]: """Find nodes that appear in the most paths.""" paths = self.get_all_paths() node_counts: Dict[str, Tuple[AttackNode, int]] = {} for path in paths: for node in path: if node.id not in node_counts: node_counts[node.id] = (node, 0) node_counts[node.id] = (node, node_counts[node.id][1] + 1) return sort
👍0
👁️0
🤖 Auto-discovered
🤖system prompt•7 months ago

web-component-design

Master React, Vue, and Svelte component patterns including

coding
⭐1
# Web Component Design Build reusable, maintainable UI components using modern frameworks with clean composition patterns and styling approaches. ## When to Use This Skill - Designing reusable component libraries or design systems - Implementing complex component composition patterns - Choosing and applying CSS-in-JS solutions - Building accessible, responsive UI components - Creating consistent component APIs across a codebase - Refactoring legacy components into modern patterns - Implementing compound components or render props ## Core Concepts ### 1. Component Composition Patterns **Compound Components**: Related components that work together ```tsx // Usage <Select value={value} onChange={setValue}> <Select.Trigger>Choose option</Select.Trigger> <Select.Options> <Select.Option value="a">Option A</Select.Option> <Select.Option value="b">Option B</Select.Option> </Select.Options> </Select> ``` **Render Props**: Delegate rendering to parent ```tsx <DataFetcher url="/api/users"> {({ data, loading, error }) => loading ? <Spinner /> : <UserList users={data} /> } </DataFetcher> ``` **Slots (Vue/Svelte)**: Named content injection points ```vue <template> <Card> <template #header>Title</template> <template #content>Body text</template> <template #footer><Button>Action</Button></template> </Card> </template> ``` ### 2. CSS-in-JS Approaches | Solution | Approach | Best For | | --------------------- | ---------------------- | --------------------------------- | | **Tailwind CSS** | Utility classes | Rapid prototyping, design systems | | **CSS Modules** | Scoped CSS files | Existing CSS, gradual adoption | | **styled-components** | Template literals | React, dynamic styling | | **Emotion** | Object/template styles | Flexible, SSR-friendly | | **Vanilla Extract** | Zero-runtime | Performance-critical apps | ### 3. Component API Design ```tsx interface ButtonProps { variant?: "primary" | "secondary" | "ghost"; size?: "sm" | "md" | "lg"; isLoading?: boolean; isDisabled?: boolean; leftIcon?: React.ReactNode; rightIcon?: React.ReactNode; children: React.ReactNode; onClick?: () => void; } ``` **Principles**: - Use semantic prop names (`isLoading` vs `loading`) - Provide sensible defaults - Support composition via `children` - Allow style overrides via `className` or `style` ## Quick Start: React Component with Tailwind ```tsx import { forwardRef, type ComponentPropsWithoutRef } from "react"; import { cva, type VariantProps } from "class-variance-authority"; import { cn } from "@/lib/utils"; const buttonVariants = cva( "inline-flex items-center justify-center rounded-md font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 disabled:pointer-events-none disabled:opacity-50", { variants: { variant: { primary: "bg-blue-600 text-white hover:bg-blue-700", secondary: "bg-gray-100 text-gray-900 hover:bg-gray-200", ghost: "hover:bg-gray-100 hover:text-gray-900", }, size: { sm: "h-8 px-3 text-sm", md: "h-10 px-4 text-sm", lg: "h-12 px-6 text-base", }, }, defaultVariants: { variant: "primary", size: "md", }, }, ); interface ButtonProps extends ComponentPropsWithoutRef<"button">, VariantProps<typeof buttonVariants> { isLoading?: boolean; } export const Button = forwardRef<HTMLButtonElement, ButtonProps>( ({ className, variant, size, isLoading, children, ...props }, ref) => ( <button ref={ref} className={cn(buttonVariants({ variant, size }), className)} disabled={isLoading || props.disabled} {...props} > {isLoading && <Spinner className="mr-2 h-4 w-4" />} {children} </button> ), ); Button.displayName = "Button"; ``` ## Framework Patterns ### React: Compound Components ```tsx import { createContext, useContext, useState, type ReactNode } from "react"; interface AccordionContextValue { openItems: Set<string>; toggle: (id: string) => void; } const AccordionContext = createContext<AccordionContextValue | null>(null); function useAccordion() { const context = useContext(AccordionContext); if (!context) throw new Error("Must be used within Accordion"); return context; } export function Accordion({ children }: { children: ReactNode }) { const [openItems, setOpenItems] = useState<Set<string>>(new Set()); const toggle = (id: string) => { setOpenItems((prev) => { const next = new Set(prev); next.has(id) ? next.delete(id) : next.add(id); return next; }); }; return ( <AccordionContext.Provider value={{ openItems, toggle }}> <div className="divide-y">{children}</div> </AccordionContext.Provider> ); } Accordion.Item = function AccordionItem({ id, title, children, }: { id: string; title: string; children: ReactNode; }) { const { openItems, toggle } = useAccordion(); const isOpen = openItems.has(id); return ( <div> <button onClick={() => toggle(id)} className="w-full text-left py-3"> {title} </button> {isOpen && <div className="pb-3">{children}</div>} </div> ); }; ``` ### Vue 3: Composables ```vue <script setup lang="ts"> import { ref, computed, provide, inject, type InjectionKey } from "vue"; interface TabsContext { activeTab: Ref<string>; setActive: (id: string) => void; } const TabsKey: InjectionKey<TabsContext> = Symbol("tabs"); // Parent component const activeTab = ref("tab-1"); provide(TabsKey, { activeTab, setActive: (id: string) => { activeTab.value = id; }, }); // Child component usage const tabs = inject(TabsKey); const isActive = computed(() => tabs?.activeTab.value === props.id); </script> ``` ### Svelte 5: Runes ```svelte <script lang="ts"> interface Props { variant?: 'primary' | 'secondary'; size?: 'sm' | 'md' | 'lg'; onclick?: () => void; children: import('svelte').Snippet; } let { variant = 'primary', size = 'md', onclick, children }: Props = $props(); const classes = $derived( `btn btn-${variant} btn-${size}` ); </script> <button class={classes} {onclick}> {@render children()} </button> ``` ## Best Practices 1. **Single Responsibility**: Each component does one thing well 2. **Prop Drilling Prevention**: Use context for deeply nested data 3. **Accessible by Default**: Include ARIA attributes, keyboard support 4. **Controlled vs Uncontrolled**: Support both patterns when appropriate 5. **Forward Refs**: Allow parent access to DOM nodes 6. **Memoization**: Use `React.memo`, `useMemo` for expensive renders 7. **Error Boundaries**: Wrap components that may fail ## Common Issues - **Prop Explosion**: Too many props - consider composition instead - **Style Conflicts**: Use scoped styles or CSS Modules - **Re-render Cascades**: Profile with React DevTools, memo appropriately - **Accessibility Gaps**: Test with screen readers and keyboard navigation - **Bundle Size**: Tree-shake unused component variants ## Resources - [React Component Patterns](https://reactpatterns.com/) - [Vue Composition API Guide](https://vuejs.org/guide/reusability/composables.html) - [Svelte Component Documentation](https://svelte.dev/docs/svelte-components) - [Radix UI Primitives](https://www.radix-ui.com/primitives) - [shadcn/ui Components](https://ui.shadcn.com/)
👍0
👁️0
🤖 Auto-discovered