Skip to main content
EVOKORE// BROWSE
>

./browse/prompts

19 NODES
šŸ¤–system prompt•7 months ago

defi-protocol-templates

Implement DeFi protocols with production-ready templates for

coding
⭐1
# DeFi Protocol Templates Production-ready templates for common DeFi protocols including staking, AMMs, governance, lending, and flash loans. ## When to Use This Skill - Building staking platforms with reward distribution - Implementing AMM (Automated Market Maker) protocols - Creating governance token systems - Developing lending/borrowing protocols - Integrating flash loan functionality - Launching yield farming platforms ## Staking Contract ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC20/IERC20.sol"; import "@openzeppelin/contracts/security/ReentrancyGuard.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; contract StakingRewards is ReentrancyGuard, Ownable { IERC20 public stakingToken; IERC20 public rewardsToken; uint256 public rewardRate = 100; // Rewards per second uint256 public lastUpdateTime; uint256 public rewardPerTokenStored; mapping(address => uint256) public userRewardPerTokenPaid; mapping(address => uint256) public rewards; mapping(address => uint256) public balances; uint256 private _totalSupply; event Staked(address indexed user, uint256 amount); event Withdrawn(address indexed user, uint256 amount); event RewardPaid(address indexed user, uint256 reward); constructor(address _stakingToken, address _rewardsToken) { stakingToken = IERC20(_stakingToken); rewardsToken = IERC20(_rewardsToken); } modifier updateReward(address account) { rewardPerTokenStored = rewardPerToken(); lastUpdateTime = block.timestamp; if (account != address(0)) { rewards[account] = earned(account); userRewardPerTokenPaid[account] = rewardPerTokenStored; } _; } function rewardPerToken() public view returns (uint256) { if (_totalSupply == 0) { return rewardPerTokenStored; } return rewardPerTokenStored + ((block.timestamp - lastUpdateTime) * rewardRate * 1e18) / _totalSupply; } function earned(address account) public view returns (uint256) { return (balances[account] * (rewardPerToken() - userRewardPerTokenPaid[account])) / 1e18 + rewards[account]; } function stake(uint256 amount) external nonReentrant updateReward(msg.sender) { require(amount > 0, "Cannot stake 0"); _totalSupply += amount; balances[msg.sender] += amount; stakingToken.transferFrom(msg.sender, address(this), amount); emit Staked(msg.sender, amount); } function withdraw(uint256 amount) public nonReentrant updateReward(msg.sender) { require(amount > 0, "Cannot withdraw 0"); _totalSupply -= amount; balances[msg.sender] -= amount; stakingToken.transfer(msg.sender, amount); emit Withdrawn(msg.sender, amount); } function getReward() public nonReentrant updateReward(msg.sender) { uint256 reward = rewards[msg.sender]; if (reward > 0) { rewards[msg.sender] = 0; rewardsToken.transfer(msg.sender, reward); emit RewardPaid(msg.sender, reward); } } function exit() external { withdraw(balances[msg.sender]); getReward(); } } ``` ## AMM (Automated Market Maker) ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC20/IERC20.sol"; contract SimpleAMM { IERC20 public token0; IERC20 public token1; uint256 public reserve0; uint256 public reserve1; uint256 public totalSupply; mapping(address => uint256) public balanceOf; event Mint(address indexed to, uint256 amount); event Burn(address indexed from, uint256 amount); event Swap(address indexed trader, uint256 amount0In, uint256 amount1In, uint256 amount0Out, uint256 amount1Out); constructor(address _token0, address _token1) { token0 = IERC20(_token0); token1 = IERC20(_token1); } function addLiquidity(uint256 amount0, uint256 amount1) external returns (uint256 shares) { token0.transferFrom(msg.sender, address(this), amount0); token1.transferFrom(msg.sender, address(this), amount1); if (totalSupply == 0) { shares = sqrt(amount0 * amount1); } else { shares = min( (amount0 * totalSupply) / reserve0, (amount1 * totalSupply) / reserve1 ); } require(shares > 0, "Shares = 0"); _mint(msg.sender, shares); _update( token0.balanceOf(address(this)), token1.balanceOf(address(this)) ); emit Mint(msg.sender, shares); } function removeLiquidity(uint256 shares) external returns (uint256 amount0, uint256 amount1) { uint256 bal0 = token0.balanceOf(address(this)); uint256 bal1 = token1.balanceOf(address(this)); amount0 = (shares * bal0) / totalSupply; amount1 = (shares * bal1) / totalSupply; require(amount0 > 0 && amount1 > 0, "Amount0 or amount1 = 0"); _burn(msg.sender, shares); _update(bal0 - amount0, bal1 - amount1); token0.transfer(msg.sender, amount0); token1.transfer(msg.sender, amount1); emit Burn(msg.sender, shares); } function swap(address tokenIn, uint256 amountIn) external returns (uint256 amountOut) { require(tokenIn == address(token0) || tokenIn == address(token1), "Invalid token"); bool isToken0 = tokenIn == address(token0); (IERC20 tokenIn_, IERC20 tokenOut, uint256 resIn, uint256 resOut) = isToken0 ? (token0, token1, reserve0, reserve1) : (token1, token0, reserve1, reserve0); tokenIn_.transferFrom(msg.sender, address(this), amountIn); // 0.3% fee uint256 amountInWithFee = (amountIn * 997) / 1000; amountOut = (resOut * amountInWithFee) / (resIn + amountInWithFee); tokenOut.transfer(msg.sender, amountOut); _update( token0.balanceOf(address(this)), token1.balanceOf(address(this)) ); emit Swap(msg.sender, isToken0 ? amountIn : 0, isToken0 ? 0 : amountIn, isToken0 ? 0 : amountOut, isToken0 ? amountOut : 0); } function _mint(address to, uint256 amount) private { balanceOf[to] += amount; totalSupply += amount; } function _burn(address from, uint256 amount) private { balanceOf[from] -= amount; totalSupply -= amount; } function _update(uint256 res0, uint256 res1) private { reserve0 = res0; reserve1 = res1; } function sqrt(uint256 y) private pure returns (uint256 z) { if (y > 3) { z = y; uint256 x = y / 2 + 1; while (x < z) { z = x; x = (y / x + x) / 2; } } else if (y != 0) { z = 1; } } function min(uint256 x, uint256 y) private pure returns (uint256) { return x <= y ? x : y; } } ``` ## Governance Token ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC20/extensions/ERC20Votes.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; contract GovernanceToken is ERC20Votes, Ownable { constructor() ERC20("Governance Token", "GOV") ERC20Permit("Governance Token") { _mint(msg.sender, 1000000 * 10**decimals()); } function _afterTokenTransfer( address from, address to, uint256 amount ) internal override(ERC20Votes) { super._afterTokenTransfer(from, to, amount); } function _mint(address to, uint256 amount) internal override(ERC20Votes) { super._mint(to, amount); } function _burn(address account, uint256 amount) internal override(ERC20Votes) { super._burn(account, amount); } } contract Governor is Ownable { GovernanceToken public governanceToken; struct Proposal { uint256 id; address proposer; string description; uint256 forVotes; uint256 againstVotes; uint256 startBlock; uint256 endBlock; bool executed; mapping(address => bool) hasVoted; } uint256 public proposalCount; mapping(uint256 => Proposal) public proposals; uint256 public votingPeriod = 17280; // ~3 days in blocks uint256 public proposalThreshold = 100000 * 10**18; event ProposalCreated(uint256 indexed proposalId, address proposer, string description); event VoteCast(address indexed voter, uint256 indexed proposalId, bool support, uint256 weight); event ProposalExecuted(uint256 indexed proposalId); constructor(address _governanceToken) { governanceToken = GovernanceToken(_governanceToken); } function propose(string memory description) external returns (uint256) { require( governanceToken.getPastVotes(msg.sender, block.number - 1) >= proposalThreshold, "Proposer votes below threshold" ); proposalCount++; Proposal storage newProposal = proposals[proposalCount]; newProposal.id = proposalCount; newProposal.proposer = msg.sender; newProposal.description = description; newProposal.startBlock = block.number; newProposal.endBlock = block.number + votingPeriod; emit ProposalCreated(proposalCount, msg.sender, description); return proposalCount; } function vote(uint256 proposalId, bool support) external { Proposal storage proposal = proposals[proposalId]; require(block.number >= proposal.startBlock, "Voting not started"); require(block.number <= proposal.endBlock, "Voting ended"); require(!proposal.hasVoted[msg.sender], "Already voted"); uint256 weight = governanceToken.getPastVotes(msg.sender, proposal.startBlock); require(weight > 0, "No voting power"); proposal.hasVoted[msg.sender] = true; if (support) { proposal.forVotes += weight; } else { proposal.againstVotes += weight; } emit VoteCast(msg.sender, proposalId, support, weight); } function execute(uint256 proposalId) external { Proposal storage proposal = proposals[proposalId]; require(block.number > proposal.endBlock, "Voting not ended"); require(!proposal.executed, "Already executed"); require(proposal.forVotes > proposal.againstVotes, "Proposal failed"); proposal.executed = true; // Execute proposal logic here emit ProposalExecuted(proposalId); } } ``` ## Flash Loan ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC20/IERC20.sol"; interface IFlashLoanReceiver { function executeOperation( address asset, uint256 amount, uint256 fee, bytes calldata params ) external returns (bool); } contract FlashLoanProvider { IERC20 public token; uint256 public feePercentage = 9; // 0.09% fee event FlashLoan(address indexed borrower, uint256 amount, uint256 fee); constructor(address _token) { token = IERC20(_token); } function flashLoan( address receiver, uint256 amount, bytes calldata params ) external { uint256 balanceBefore = token.balanceOf(address(this)); require(balanceBefore >= amount, "Insufficient liquidity"); uint256 fee = (amount * feePercentage) / 10000; // Send tokens to receiver token.transfer(receiver, amount); // Execute callback require( IFlashLoanReceiver(receiver).executeOperation( address(token), amount, fee, params ), "Flash loan failed" ); // Verify repayment uint256 balanceAfter = token.balanceOf(address(this)); require(balanceAfter >= balanceBefore + fee, "Flash loan not repaid"); emit FlashLoan(receiver, amount, fee); } } // Example flash loan receiver contract FlashLoanReceiver is IFlashLoanReceiver { function executeOperation( address asset, uint256 amount, uint256 fee, bytes calldata params ) external override returns (bool) { // Decode params and execute arbitrage, liquidation, etc. // ... // Approve repayment IERC20(asset).approve(msg.sender, amount + fee); return true; } } ``` ## Resources - **references/staking.md**: Staking mechanics and reward distribution - **references/liquidity-pools.md**: AMM mathematics and pricing - **references/governance-tokens.md**: Governance and voting systems - **references/lending-protocols.md**: Lending/borrowing implementation - **references/flash-loans.md**: Flash loan security and use cases - **assets/staking-contract.sol**: Production staking template - **assets/amm-contract.sol**: Full AMM implementation - **assets/governance-token.sol**: Governance system - **assets/lending-protocol.sol**: Lending platform template ## Best Practices 1. **Use Established Libraries**: OpenZeppelin, Solmate 2. **Test Thoroughly**: Unit tests, integration tests, fuzzing 3. **Audit Before Launch**: Professional security audits 4. **Start Simple**: MVP first, add features incrementally 5. **Monitor**: Track contract health and user activity 6. **Upgradability**: Consider proxy patterns for upgrades 7. **Emergency Controls**: Pause mechanisms for critical issues ## Common DeFi Patterns - **Time-Weighted Average Price (TWAP)**: Price oracle resistance - **Liquidity Mining**: Incentivize liquidity provision - **Vesting**: Lock tokens with gradual release - **Multisig**: Require multiple signatures for critical operations - **Timelocks**: Delay execution of governance decisions
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

nft-standards

Implement NFT standards (ERC-721, ERC-1155) with proper metadata

coding
⭐1
# NFT Standards Master ERC-721 and ERC-1155 NFT standards, metadata best practices, and advanced NFT features. ## When to Use This Skill - Creating NFT collections (art, gaming, collectibles) - Implementing marketplace functionality - Building on-chain or off-chain metadata - Creating soulbound tokens (non-transferable) - Implementing royalties and revenue sharing - Developing dynamic/evolving NFTs ## ERC-721 (Non-Fungible Token Standard) ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC721/extensions/ERC721URIStorage.sol"; import "@openzeppelin/contracts/token/ERC721/extensions/ERC721Enumerable.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; import "@openzeppelin/contracts/utils/Counters.sol"; contract MyNFT is ERC721URIStorage, ERC721Enumerable, Ownable { using Counters for Counters.Counter; Counters.Counter private _tokenIds; uint256 public constant MAX_SUPPLY = 10000; uint256 public constant MINT_PRICE = 0.08 ether; uint256 public constant MAX_PER_MINT = 20; constructor() ERC721("MyNFT", "MNFT") {} function mint(uint256 quantity) external payable { require(quantity > 0 && quantity <= MAX_PER_MINT, "Invalid quantity"); require(_tokenIds.current() + quantity <= MAX_SUPPLY, "Exceeds max supply"); require(msg.value >= MINT_PRICE * quantity, "Insufficient payment"); for (uint256 i = 0; i < quantity; i++) { _tokenIds.increment(); uint256 newTokenId = _tokenIds.current(); _safeMint(msg.sender, newTokenId); _setTokenURI(newTokenId, generateTokenURI(newTokenId)); } } function generateTokenURI(uint256 tokenId) internal pure returns (string memory) { // Return IPFS URI or on-chain metadata return string(abi.encodePacked("ipfs://QmHash/", Strings.toString(tokenId), ".json")); } // Required overrides function _beforeTokenTransfer( address from, address to, uint256 tokenId, uint256 batchSize ) internal override(ERC721, ERC721Enumerable) { super._beforeTokenTransfer(from, to, tokenId, batchSize); } function _burn(uint256 tokenId) internal override(ERC721, ERC721URIStorage) { super._burn(tokenId); } function tokenURI(uint256 tokenId) public view override(ERC721, ERC721URIStorage) returns (string memory) { return super.tokenURI(tokenId); } function supportsInterface(bytes4 interfaceId) public view override(ERC721, ERC721Enumerable) returns (bool) { return super.supportsInterface(interfaceId); } function withdraw() external onlyOwner { payable(owner()).transfer(address(this).balance); } } ``` ## ERC-1155 (Multi-Token Standard) ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "@openzeppelin/contracts/token/ERC1155/ERC1155.sol"; import "@openzeppelin/contracts/access/Ownable.sol"; contract GameItems is ERC1155, Ownable { uint256 public constant SWORD = 1; uint256 public constant SHIELD = 2; uint256 public constant POTION = 3; mapping(uint256 => uint256) public tokenSupply; mapping(uint256 => uint256) public maxSupply; constructor() ERC1155("ipfs://QmBaseHash/{id}.json") { maxSupply[SWORD] = 1000; maxSupply[SHIELD] = 500; maxSupply[POTION] = 10000; } function mint( address to, uint256 id, uint256 amount ) external onlyOwner { require(tokenSupply[id] + amount <= maxSupply[id], "Exceeds max supply"); _mint(to, id, amount, ""); tokenSupply[id] += amount; } function mintBatch( address to, uint256[] memory ids, uint256[] memory amounts ) external onlyOwner { for (uint256 i = 0; i < ids.length; i++) { require(tokenSupply[ids[i]] + amounts[i] <= maxSupply[ids[i]], "Exceeds max supply"); tokenSupply[ids[i]] += amounts[i]; } _mintBatch(to, ids, amounts, ""); } function burn( address from, uint256 id, uint256 amount ) external { require(from == msg.sender || isApprovedForAll(from, msg.sender), "Not authorized"); _burn(from, id, amount); tokenSupply[id] -= amount; } } ``` ## Metadata Standards ### Off-Chain Metadata (IPFS) ```json { "name": "NFT #1", "description": "Description of the NFT", "image": "ipfs://QmImageHash", "attributes": [ { "trait_type": "Background", "value": "Blue" }, { "trait_type": "Rarity", "value": "Legendary" }, { "trait_type": "Power", "value": 95, "display_type": "number", "max_value": 100 } ] } ``` ### On-Chain Metadata ```solidity contract OnChainNFT is ERC721 { struct Traits { uint8 background; uint8 body; uint8 head; uint8 rarity; } mapping(uint256 => Traits) public tokenTraits; function tokenURI(uint256 tokenId) public view override returns (string memory) { Traits memory traits = tokenTraits[tokenId]; string memory json = Base64.encode( bytes( string( abi.encodePacked( '{"name": "NFT #', Strings.toString(tokenId), '",', '"description": "On-chain NFT",', '"image": "data:image/svg+xml;base64,', generateSVG(traits), '",', '"attributes": [', '{"trait_type": "Background", "value": "', Strings.toString(traits.background), '"},', '{"trait_type": "Rarity", "value": "', getRarityName(traits.rarity), '"}', ']}' ) ) ) ); return string(abi.encodePacked("data:application/json;base64,", json)); } function generateSVG(Traits memory traits) internal pure returns (string memory) { // Generate SVG based on traits return "..."; } } ``` ## Royalties (EIP-2981) ```solidity import "@openzeppelin/contracts/interfaces/IERC2981.sol"; contract NFTWithRoyalties is ERC721, IERC2981 { address public royaltyRecipient; uint96 public royaltyFee = 500; // 5% constructor() ERC721("Royalty NFT", "RNFT") { royaltyRecipient = msg.sender; } function royaltyInfo(uint256 tokenId, uint256 salePrice) external view override returns (address receiver, uint256 royaltyAmount) { return (royaltyRecipient, (salePrice * royaltyFee) / 10000); } function setRoyalty(address recipient, uint96 fee) external onlyOwner { require(fee <= 1000, "Royalty fee too high"); // Max 10% royaltyRecipient = recipient; royaltyFee = fee; } function supportsInterface(bytes4 interfaceId) public view override(ERC721, IERC165) returns (bool) { return interfaceId == type(IERC2981).interfaceId || super.supportsInterface(interfaceId); } } ``` ## Soulbound Tokens (Non-Transferable) ```solidity contract SoulboundToken is ERC721 { constructor() ERC721("Soulbound", "SBT") {} function _beforeTokenTransfer( address from, address to, uint256 tokenId, uint256 batchSize ) internal virtual override { require(from == address(0) || to == address(0), "Token is soulbound"); super._beforeTokenTransfer(from, to, tokenId, batchSize); } function mint(address to) external { uint256 tokenId = totalSupply() + 1; _safeMint(to, tokenId); } // Burn is allowed (user can destroy their SBT) function burn(uint256 tokenId) external { require(ownerOf(tokenId) == msg.sender, "Not token owner"); _burn(tokenId); } } ``` ## Dynamic NFTs ```solidity contract DynamicNFT is ERC721 { struct TokenState { uint256 level; uint256 experience; uint256 lastUpdated; } mapping(uint256 => TokenState) public tokenStates; function gainExperience(uint256 tokenId, uint256 exp) external { require(ownerOf(tokenId) == msg.sender, "Not token owner"); TokenState storage state = tokenStates[tokenId]; state.experience += exp; // Level up logic if (state.experience >= state.level * 100) { state.level++; } state.lastUpdated = block.timestamp; } function tokenURI(uint256 tokenId) public view override returns (string memory) { TokenState memory state = tokenStates[tokenId]; // Generate metadata based on current state return generateMetadata(tokenId, state); } function generateMetadata(uint256 tokenId, TokenState memory state) internal pure returns (string memory) { // Dynamic metadata generation return ""; } } ``` ## Gas-Optimized Minting (ERC721A) ```solidity import "erc721a/contracts/ERC721A.sol"; contract OptimizedNFT is ERC721A { uint256 public constant MAX_SUPPLY = 10000; uint256 public constant MINT_PRICE = 0.05 ether; constructor() ERC721A("Optimized NFT", "ONFT") {} function mint(uint256 quantity) external payable { require(_totalMinted() + quantity <= MAX_SUPPLY, "Exceeds max supply"); require(msg.value >= MINT_PRICE * quantity, "Insufficient payment"); _mint(msg.sender, quantity); } function _baseURI() internal pure override returns (string memory) { return "ipfs://QmBaseHash/"; } } ``` ## Resources - **references/erc721.md**: ERC-721 specification details - **references/erc1155.md**: ERC-1155 multi-token standard - **references/metadata-standards.md**: Metadata best practices - **references/enumeration.md**: Token enumeration patterns - **assets/erc721-contract.sol**: Production ERC-721 template - **assets/erc1155-contract.sol**: Production ERC-1155 template - **assets/metadata-schema.json**: Standard metadata format - **assets/metadata-uploader.py**: IPFS upload utility ## Best Practices 1. **Use OpenZeppelin**: Battle-tested implementations 2. **Pin Metadata**: Use IPFS with pinning service 3. **Implement Royalties**: EIP-2981 for marketplace compatibility 4. **Gas Optimization**: Use ERC721A for batch minting 5. **Reveal Mechanism**: Placeholder → reveal pattern 6. **Enumeration**: Support walletOfOwner for marketplaces 7. **Whitelist**: Merkle trees for efficient whitelisting ## Marketplace Integration - OpenSea: ERC-721/1155, metadata standards - LooksRare: Royalty enforcement - Rarible: Protocol fees, lazy minting - Blur: Gas-optimized trading
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

web3-testing

Test smart contracts comprehensively using Hardhat and Foundry with

coding
⭐1
# Web3 Smart Contract Testing Master comprehensive testing strategies for smart contracts using Hardhat, Foundry, and advanced testing patterns. ## When to Use This Skill - Writing unit tests for smart contracts - Setting up integration test suites - Performing gas optimization testing - Fuzzing for edge cases - Forking mainnet for realistic testing - Automating test coverage reporting - Verifying contracts on Etherscan ## Hardhat Testing Setup ```javascript // hardhat.config.js require("@nomicfoundation/hardhat-toolbox"); require("@nomiclabs/hardhat-etherscan"); require("hardhat-gas-reporter"); require("solidity-coverage"); module.exports = { solidity: { version: "0.8.19", settings: { optimizer: { enabled: true, runs: 200, }, }, }, networks: { hardhat: { forking: { url: process.env.MAINNET_RPC_URL, blockNumber: 15000000, }, }, goerli: { url: process.env.GOERLI_RPC_URL, accounts: [process.env.PRIVATE_KEY], }, }, gasReporter: { enabled: true, currency: "USD", coinmarketcap: process.env.COINMARKETCAP_API_KEY, }, etherscan: { apiKey: process.env.ETHERSCAN_API_KEY, }, }; ``` ## Unit Testing Patterns ```javascript const { expect } = require("chai"); const { ethers } = require("hardhat"); const { loadFixture, time, } = require("@nomicfoundation/hardhat-network-helpers"); describe("Token Contract", function () { // Fixture for test setup async function deployTokenFixture() { const [owner, addr1, addr2] = await ethers.getSigners(); const Token = await ethers.getContractFactory("Token"); const token = await Token.deploy(); return { token, owner, addr1, addr2 }; } describe("Deployment", function () { it("Should set the right owner", async function () { const { token, owner } = await loadFixture(deployTokenFixture); expect(await token.owner()).to.equal(owner.address); }); it("Should assign total supply to owner", async function () { const { token, owner } = await loadFixture(deployTokenFixture); const ownerBalance = await token.balanceOf(owner.address); expect(await token.totalSupply()).to.equal(ownerBalance); }); }); describe("Transactions", function () { it("Should transfer tokens between accounts", async function () { const { token, owner, addr1 } = await loadFixture(deployTokenFixture); await expect(token.transfer(addr1.address, 50)).to.changeTokenBalances( token, [owner, addr1], [-50, 50], ); }); it("Should fail if sender doesn't have enough tokens", async function () { const { token, addr1 } = await loadFixture(deployTokenFixture); const initialBalance = await token.balanceOf(addr1.address); await expect( token.connect(addr1).transfer(owner.address, 1), ).to.be.revertedWith("Insufficient balance"); }); it("Should emit Transfer event", async function () { const { token, owner, addr1 } = await loadFixture(deployTokenFixture); await expect(token.transfer(addr1.address, 50)) .to.emit(token, "Transfer") .withArgs(owner.address, addr1.address, 50); }); }); describe("Time-based tests", function () { it("Should handle time-locked operations", async function () { const { token } = await loadFixture(deployTokenFixture); // Increase time by 1 day await time.increase(86400); // Test time-dependent functionality }); }); describe("Gas optimization", function () { it("Should use gas efficiently", async function () { const { token } = await loadFixture(deployTokenFixture); const tx = await token.transfer(addr1.address, 100); const receipt = await tx.wait(); expect(receipt.gasUsed).to.be.lessThan(50000); }); }); }); ``` ## Foundry Testing (Forge) ```solidity // SPDX-License-Identifier: MIT pragma solidity ^0.8.0; import "forge-std/Test.sol"; import "../src/Token.sol"; contract TokenTest is Test { Token token; address owner = address(1); address user1 = address(2); address user2 = address(3); function setUp() public { vm.prank(owner); token = new Token(); } function testInitialSupply() public { assertEq(token.totalSupply(), 1000000 * 10**18); } function testTransfer() public { vm.prank(owner); token.transfer(user1, 100); assertEq(token.balanceOf(user1), 100); assertEq(token.balanceOf(owner), token.totalSupply() - 100); } function testFailTransferInsufficientBalance() public { vm.prank(user1); token.transfer(user2, 100); // Should fail } function testCannotTransferToZeroAddress() public { vm.prank(owner); vm.expectRevert("Invalid recipient"); token.transfer(address(0), 100); } // Fuzzing test function testFuzzTransfer(uint256 amount) public { vm.assume(amount > 0 && amount <= token.totalSupply()); vm.prank(owner); token.transfer(user1, amount); assertEq(token.balanceOf(user1), amount); } // Test with cheatcodes function testDealAndPrank() public { // Give ETH to address vm.deal(user1, 10 ether); // Impersonate address vm.prank(user1); // Test functionality assertEq(user1.balance, 10 ether); } // Mainnet fork test function testForkMainnet() public { vm.createSelectFork("https://eth-mainnet.alchemyapi.io/v2/..."); // Interact with mainnet contracts address dai = 0x6B175474E89094C44Da98b954EedeAC495271d0F; assertEq(IERC20(dai).symbol(), "DAI"); } } ``` ## Advanced Testing Patterns ### Snapshot and Revert ```javascript describe("Complex State Changes", function () { let snapshotId; beforeEach(async function () { snapshotId = await network.provider.send("evm_snapshot"); }); afterEach(async function () { await network.provider.send("evm_revert", [snapshotId]); }); it("Test 1", async function () { // Make state changes }); it("Test 2", async function () { // State reverted, clean slate }); }); ``` ### Mainnet Forking ```javascript describe("Mainnet Fork Tests", function () { let uniswapRouter, dai, usdc; before(async function () { await network.provider.request({ method: "hardhat_reset", params: [ { forking: { jsonRpcUrl: process.env.MAINNET_RPC_URL, blockNumber: 15000000, }, }, ], }); // Connect to existing mainnet contracts uniswapRouter = await ethers.getContractAt( "IUniswapV2Router", "0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D", ); dai = await ethers.getContractAt( "IERC20", "0x6B175474E89094C44Da98b954EedeAC495271d0F", ); }); it("Should swap on Uniswap", async function () { // Test with real Uniswap contracts }); }); ``` ### Impersonating Accounts ```javascript it("Should impersonate whale account", async function () { const whaleAddress = "0x..."; await network.provider.request({ method: "hardhat_impersonateAccount", params: [whaleAddress], }); const whale = await ethers.getSigner(whaleAddress); // Use whale's tokens await dai .connect(whale) .transfer(addr1.address, ethers.utils.parseEther("1000")); }); ``` ## Gas Optimization Testing ```javascript const { expect } = require("chai"); describe("Gas Optimization", function () { it("Compare gas usage between implementations", async function () { const Implementation1 = await ethers.getContractFactory("OptimizedContract"); const Implementation2 = await ethers.getContractFactory( "UnoptimizedContract", ); const contract1 = await Implementation1.deploy(); const contract2 = await Implementation2.deploy(); const tx1 = await contract1.doSomething(); const receipt1 = await tx1.wait(); const tx2 = await contract2.doSomething(); const receipt2 = await tx2.wait(); console.log("Optimized gas:", receipt1.gasUsed.toString()); console.log("Unoptimized gas:", receipt2.gasUsed.toString()); expect(receipt1.gasUsed).to.be.lessThan(receipt2.gasUsed); }); }); ``` ## Coverage Reporting ```bash # Generate coverage report npx hardhat coverage # Output shows: # File | % Stmts | % Branch | % Funcs | % Lines | # -------------------|---------|----------|---------|---------| # contracts/Token.sol | 100 | 90 | 100 | 95 | ``` ## Contract Verification ```javascript // Verify on Etherscan await hre.run("verify:verify", { address: contractAddress, constructorArguments: [arg1, arg2], }); ``` ```bash # Or via CLI npx hardhat verify --network mainnet CONTRACT_ADDRESS "Constructor arg1" "arg2" ``` ## CI/CD Integration ```yaml # .github/workflows/test.yml name: Tests on: [push, pull_request] jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: node-version: "16" - run: npm install - run: npx hardhat compile - run: npx hardhat test - run: npx hardhat coverage - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 ``` ## Resources - **references/hardhat-setup.md**: Hardhat configuration guide - **references/foundry-setup.md**: Foundry testing framework - **references/test-patterns.md**: Testing best practices - **references/mainnet-forking.md**: Fork testing strategies - **references/contract-verification.md**: Etherscan verification - **assets/hardhat-config.js**: Complete Hardhat configuration - **assets/test-suite.js**: Comprehensive test examples - **assets/foundry.toml**: Foundry configuration - **scripts/test-contract.sh**: Automated testing script ## Best Practices 1. **Test Coverage**: Aim for >90% coverage 2. **Edge Cases**: Test boundary conditions 3. **Gas Limits**: Verify functions don't hit block gas limit 4. **Reentrancy**: Test for reentrancy vulnerabilities 5. **Access Control**: Test unauthorized access attempts 6. **Events**: Verify event emissions 7. **Fixtures**: Use fixtures to avoid code duplication 8. **Mainnet Fork**: Test with real contracts 9. **Fuzzing**: Use property-based testing 10. **CI/CD**: Automate testing on every commit
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

e2e-testing-patterns

Master end-to-end testing with Playwright and Cypress to build

coding
⭐1
# E2E Testing Patterns Build reliable, fast, and maintainable end-to-end test suites that provide confidence to ship code quickly and catch regressions before users do. ## When to Use This Skill - Implementing end-to-end test automation - Debugging flaky or unreliable tests - Testing critical user workflows - Setting up CI/CD test pipelines - Testing across multiple browsers - Validating accessibility requirements - Testing responsive designs - Establishing E2E testing standards ## Core Concepts ### 1. E2E Testing Fundamentals **What to Test with E2E:** - Critical user journeys (login, checkout, signup) - Complex interactions (drag-and-drop, multi-step forms) - Cross-browser compatibility - Real API integration - Authentication flows **What NOT to Test with E2E:** - Unit-level logic (use unit tests) - API contracts (use integration tests) - Edge cases (too slow) - Internal implementation details ### 2. Test Philosophy **The Testing Pyramid:** ``` /\ /E2E\ ← Few, focused on critical paths /─────\ /Integr\ ← More, test component interactions /────────\ /Unit Tests\ ← Many, fast, isolated /────────────\ ``` **Best Practices:** - Test user behavior, not implementation - Keep tests independent - Make tests deterministic - Optimize for speed - Use data-testid, not CSS selectors ## Playwright Patterns ### Setup and Configuration ```typescript // playwright.config.ts import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./e2e", timeout: 30000, expect: { timeout: 5000, }, fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, workers: process.env.CI ? 1 : undefined, reporter: [["html"], ["junit", { outputFile: "results.xml" }]], use: { baseURL: "http://localhost:3000", trace: "on-first-retry", screenshot: "only-on-failure", video: "retain-on-failure", }, projects: [ { name: "chromium", use: { ...devices["Desktop Chrome"] } }, { name: "firefox", use: { ...devices["Desktop Firefox"] } }, { name: "webkit", use: { ...devices["Desktop Safari"] } }, { name: "mobile", use: { ...devices["iPhone 13"] } }, ], }); ``` ### Pattern 1: Page Object Model ```typescript // pages/LoginPage.ts import { Page, Locator } from "@playwright/test"; export class LoginPage { readonly page: Page; readonly emailInput: Locator; readonly passwordInput: Locator; readonly loginButton: Locator; readonly errorMessage: Locator; constructor(page: Page) { this.page = page; this.emailInput = page.getByLabel("Email"); this.passwordInput = page.getByLabel("Password"); this.loginButton = page.getByRole("button", { name: "Login" }); this.errorMessage = page.getByRole("alert"); } async goto() { await this.page.goto("/login"); } async login(email: string, password: string) { await this.emailInput.fill(email); await this.passwordInput.fill(password); await this.loginButton.click(); } async getErrorMessage(): Promise<string> { return (await this.errorMessage.textContent()) ?? ""; } } // Test using Page Object import { test, expect } from "@playwright/test"; import { LoginPage } from "./pages/LoginPage"; test("successful login", async ({ page }) => { const loginPage = new LoginPage(page); await loginPage.goto(); await loginPage.login("user@example.com", "password123"); await expect(page).toHaveURL("/dashboard"); await expect(page.getByRole("heading", { name: "Dashboard" })).toBeVisible(); }); test("failed login shows error", async ({ page }) => { const loginPage = new LoginPage(page); await loginPage.goto(); await loginPage.login("invalid@example.com", "wrong"); const error = await loginPage.getErrorMessage(); expect(error).toContain("Invalid credentials"); }); ``` ### Pattern 2: Fixtures for Test Data ```typescript // fixtures/test-data.ts import { test as base } from "@playwright/test"; type TestData = { testUser: { email: string; password: string; name: string; }; adminUser: { email: string; password: string; }; }; export const test = base.extend<TestData>({ testUser: async ({}, use) => { const user = { email: `test-${Date.now()}@example.com`, password: "Test123!@#", name: "Test User", }; // Setup: Create user in database await createTestUser(user); await use(user); // Teardown: Clean up user await deleteTestUser(user.email); }, adminUser: async ({}, use) => { await use({ email: "admin@example.com", password: process.env.ADMIN_PASSWORD!, }); }, }); // Usage in tests import { test } from "./fixtures/test-data"; test("user can update profile", async ({ page, testUser }) => { await page.goto("/login"); await page.getByLabel("Email").fill(testUser.email); await page.getByLabel("Password").fill(testUser.password); await page.getByRole("button", { name: "Login" }).click(); await page.goto("/profile"); await page.getByLabel("Name").fill("Updated Name"); await page.getByRole("button", { name: "Save" }).click(); await expect(page.getByText("Profile updated")).toBeVisible(); }); ``` ### Pattern 3: Waiting Strategies ```typescript // āŒ Bad: Fixed timeouts await page.waitForTimeout(3000); // Flaky! // āœ… Good: Wait for specific conditions await page.waitForLoadState("networkidle"); await page.waitForURL("/dashboard"); await page.waitForSelector('[data-testid="user-profile"]'); // āœ… Better: Auto-waiting with assertions await expect(page.getByText("Welcome")).toBeVisible(); await expect(page.getByRole("button", { name: "Submit" })).toBeEnabled(); // Wait for API response const responsePromise = page.waitForResponse( (response) => response.url().includes("/api/users") && response.status() === 200, ); await page.getByRole("button", { name: "Load Users" }).click(); const response = await responsePromise; const data = await response.json(); expect(data.users).toHaveLength(10); // Wait for multiple conditions await Promise.all([ page.waitForURL("/success"), page.waitForLoadState("networkidle"), expect(page.getByText("Payment successful")).toBeVisible(), ]); ``` ### Pattern 4: Network Mocking and Interception ```typescript // Mock API responses test("displays error when API fails", async ({ page }) => { await page.route("**/api/users", (route) => { route.fulfill({ status: 500, contentType: "application/json", body: JSON.stringify({ error: "Internal Server Error" }), }); }); await page.goto("/users"); await expect(page.getByText("Failed to load users")).toBeVisible(); }); // Intercept and modify requests test("can modify API request", async ({ page }) => { await page.route("**/api/users", async (route) => { const request = route.request(); const postData = JSON.parse(request.postData() || "{}"); // Modify request postData.role = "admin"; await route.continue({ postData: JSON.stringify(postData), }); }); // Test continues... }); // Mock third-party services test("payment flow with mocked Stripe", async ({ page }) => { await page.route("**/api/stripe/**", (route) => { route.fulfill({ status: 200, body: JSON.stringify({ id: "mock_payment_id", status: "succeeded", }), }); }); // Test payment flow with mocked response }); ``` ## Cypress Patterns ### Setup and Configuration ```typescript // cypress.config.ts import { defineConfig } from "cypress"; export default defineConfig({ e2e: { baseUrl: "http://localhost:3000", viewportWidth: 1280, viewportHeight: 720, video: false, screenshotOnRunFailure: true, defaultCommandTimeout: 10000, requestTimeout: 10000, setupNodeEvents(on, config) { // Implement node event listeners }, }, }); ``` ### Pattern 1: Custom Commands ```typescript // cypress/support/commands.ts declare global { namespace Cypress { interface Chainable { login(email: string, password: string): Chainable<void>; createUser(userData: UserData): Chainable<User>; dataCy(value: string): Chainable<JQuery<HTMLElement>>; } } } Cypress.Commands.add("login", (email: string, password: string) => { cy.visit("/login"); cy.get('[data-testid="email"]').type(email); cy.get('[data-testid="password"]').type(password); cy.get('[data-testid="login-button"]').click(); cy.url().should("include", "/dashboard"); }); Cypress.Commands.add("createUser", (userData: UserData) => { return cy.request("POST", "/api/users", userData).its("body"); }); Cypress.Commands.add("dataCy", (value: string) => { return cy.get(`[data-cy="${value}"]`); }); // Usage cy.login("user@example.com", "password"); cy.dataCy("submit-button").click(); ``` ### Pattern 2: Cypress Intercept ```typescript // Mock API calls cy.intercept("GET", "/api/users", { statusCode: 200, body: [ { id: 1, name: "John" }, { id: 2, name: "Jane" }, ], }).as("getUsers"); cy.visit("/users"); cy.wait("@getUsers"); cy.get('[data-testid="user-list"]').children().should("have.length", 2); // Modify responses cy.intercept("GET", "/api/users", (req) => { req.reply((res) => { // Modify response res.body.users = res.body.users.slice(0, 5); res.send(); }); }); // Simulate slow network cy.intercept("GET", "/api/data", (req) => { req.reply((res) => { res.delay(3000); // 3 second delay res.send(); }); }); ``` ## Advanced Patterns ### Pattern 1: Visual Regression Testing ```typescript // With Playwright import { test, expect } from "@playwright/test"; test("homepage looks correct", async ({ page }) => { await page.goto("/"); await expect(page).toHaveScreenshot("homepage.png", { fullPage: true, maxDiffPixels: 100, }); }); test("button in all states", async ({ page }) => { await page.goto("/components"); const button = page.getByRole("button", { name: "Submit" }); // Default state await expect(button).toHaveScreenshot("button-default.png"); // Hover state await button.hover(); await expect(button).toHaveScreenshot("button-hover.png"); // Disabled state await button.evaluate((el) => el.setAttribute("disabled", "true")); await expect(button).toHaveScreenshot("button-disabled.png"); }); ``` ### Pattern 2: Parallel Testing with Sharding ```typescript // playwright.config.ts export default defineConfig({ projects: [ { name: "shard-1", use: { ...devices["Desktop Chrome"] }, grepInvert: /@slow/, shard: { current: 1, total: 4 }, }, { name: "shard-2", use: { ...devices["Desktop Chrome"] }, shard: { current: 2, total: 4 }, }, // ... more shards ], }); // Run in CI // npx playwright test --shard=1/4 // npx playwright test --shard=2/4 ``` ### Pattern 3: Accessibility Testing ```typescript // Install: npm install @axe-core/playwright import { test, expect } from "@playwright/test"; import AxeBuilder from "@axe-core/playwright"; test("page should not have accessibility violations", async ({ page }) => { await page.goto("/"); const accessibilityScanResults = await new AxeBuilder({ page }) .exclude("#third-party-widget") .analyze(); expect(accessibilityScanResults.violations).toEqual([]); }); test("form is accessible", async ({ page }) => { await page.goto("/signup"); const results = await new AxeBuilder({ page }).include("form").analyze(); expect(results.violations).toEqual([]); }); ``` ## Best Practices 1. **Use Data Attributes**: `data-testid` or `data-cy` for stable selectors 2. **Avoid Brittle Selectors**: Don't rely on CSS classes or DOM structure 3. **Test User Behavior**: Click, type, see - not implementation details 4. **Keep Tests Independent**: Each test should run in isolation 5. **Clean Up Test Data**: Create and destroy test data in each test 6. **Use Page Objects**: Encapsulate page logic 7. **Meaningful Assertions**: Check actual user-visible behavior 8. **Optimize for Speed**: Mock when possible, parallel execution ```typescript // āŒ Bad selectors cy.get(".btn.btn-primary.submit-button").click(); cy.get("div > form > div:nth-child(2) > input").type("text"); // āœ… Good selectors cy.getByRole("button", { name: "Submit" }).click(); cy.getByLabel("Email address").type("user@example.com"); cy.get('[data-testid="email-input"]').type("user@example.com"); ``` ## Common Pitfalls - **Flaky Tests**: Use proper waits, not fixed timeouts - **Slow Tests**: Mock external APIs, use parallel execution - **Over-Testing**: Don't test every edge case with E2E - **Coupled Tests**: Tests should not depend on each other - **Poor Selectors**: Avoid CSS classes and nth-child - **No Cleanup**: Clean up test data after each test - **Testing Implementation**: Test user behavior, not internals ## Debugging Failing Tests ```typescript // Playwright debugging // 1. Run in headed mode npx playwright test --headed // 2. Run in debug mode npx playwright test --debug // 3. Use trace viewer await page.screenshot({ path: 'screenshot.png' }); await page.video()?.saveAs('video.webm'); // 4. Add test.step for better reporting test('checkout flow', async ({ page }) => { await test.step('Add item to cart', async () => { await page.goto('/products'); await page.getByRole('button', { name: 'Add to Cart' }).click(); }); await test.step('Proceed to checkout', async () => { await page.goto('/cart'); await page.getByRole('button', { name: 'Checkout' }).click(); }); }); // 5. Inspect page state await page.pause(); // Pauses execution, opens inspector ``` ## Resources - **references/playwright-best-practices.md**: Playwright-specific patterns - **references/cypress-best-practices.md**: Cypress-specific patterns - **references/flaky-test-debugging.md**: Debugging unreliable tests - **assets/e2e-testing-checklist.md**: What to test with E2E - **assets/selector-strategies.md**: Finding reliable selectors - **scripts/test-analyzer.ts**: Analyze test flakiness and duration
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

error-handling-patterns

Master error handling patterns across languages including

coding
⭐1
# Error Handling Patterns Build resilient applications with robust error handling strategies that gracefully handle failures and provide excellent debugging experiences. ## When to Use This Skill - Implementing error handling in new features - Designing error-resilient APIs - Debugging production issues - Improving application reliability - Creating better error messages for users and developers - Implementing retry and circuit breaker patterns - Handling async/concurrent errors - Building fault-tolerant distributed systems ## Core Concepts ### 1. Error Handling Philosophies **Exceptions vs Result Types:** - **Exceptions**: Traditional try-catch, disrupts control flow - **Result Types**: Explicit success/failure, functional approach - **Error Codes**: C-style, requires discipline - **Option/Maybe Types**: For nullable values **When to Use Each:** - Exceptions: Unexpected errors, exceptional conditions - Result Types: Expected errors, validation failures - Panics/Crashes: Unrecoverable errors, programming bugs ### 2. Error Categories **Recoverable Errors:** - Network timeouts - Missing files - Invalid user input - API rate limits **Unrecoverable Errors:** - Out of memory - Stack overflow - Programming bugs (null pointer, etc.) ## Language-Specific Patterns ### Python Error Handling **Custom Exception Hierarchy:** ```python class ApplicationError(Exception): """Base exception for all application errors.""" def __init__(self, message: str, code: str = None, details: dict = None): super().__init__(message) self.code = code self.details = details or {} self.timestamp = datetime.utcnow() class ValidationError(ApplicationError): """Raised when validation fails.""" pass class NotFoundError(ApplicationError): """Raised when resource not found.""" pass class ExternalServiceError(ApplicationError): """Raised when external service fails.""" def __init__(self, message: str, service: str, **kwargs): super().__init__(message, **kwargs) self.service = service # Usage def get_user(user_id: str) -> User: user = db.query(User).filter_by(id=user_id).first() if not user: raise NotFoundError( f"User not found", code="USER_NOT_FOUND", details={"user_id": user_id} ) return user ``` **Context Managers for Cleanup:** ```python from contextlib import contextmanager @contextmanager def database_transaction(session): """Ensure transaction is committed or rolled back.""" try: yield session session.commit() except Exception as e: session.rollback() raise finally: session.close() # Usage with database_transaction(db.session) as session: user = User(name="Alice") session.add(user) # Automatic commit or rollback ``` **Retry with Exponential Backoff:** ```python import time from functools import wraps from typing import TypeVar, Callable T = TypeVar('T') def retry( max_attempts: int = 3, backoff_factor: float = 2.0, exceptions: tuple = (Exception,) ): """Retry decorator with exponential backoff.""" def decorator(func: Callable[..., T]) -> Callable[..., T]: @wraps(func) def wrapper(*args, **kwargs) -> T: last_exception = None for attempt in range(max_attempts): try: return func(*args, **kwargs) except exceptions as e: last_exception = e if attempt < max_attempts - 1: sleep_time = backoff_factor ** attempt time.sleep(sleep_time) continue raise raise last_exception return wrapper return decorator # Usage @retry(max_attempts=3, exceptions=(NetworkError,)) def fetch_data(url: str) -> dict: response = requests.get(url, timeout=5) response.raise_for_status() return response.json() ``` ### TypeScript/JavaScript Error Handling **Custom Error Classes:** ```typescript // Custom error classes class ApplicationError extends Error { constructor( message: string, public code: string, public statusCode: number = 500, public details?: Record<string, any>, ) { super(message); this.name = this.constructor.name; Error.captureStackTrace(this, this.constructor); } } class ValidationError extends ApplicationError { constructor(message: string, details?: Record<string, any>) { super(message, "VALIDATION_ERROR", 400, details); } } class NotFoundError extends ApplicationError { constructor(resource: string, id: string) { super(`${resource} not found`, "NOT_FOUND", 404, { resource, id }); } } // Usage function getUser(id: string): User { const user = users.find((u) => u.id === id); if (!user) { throw new NotFoundError("User", id); } return user; } ``` **Result Type Pattern:** ```typescript // Result type for explicit error handling type Result<T, E = Error> = { ok: true; value: T } | { ok: false; error: E }; // Helper functions function Ok<T>(value: T): Result<T, never> { return { ok: true, value }; } function Err<E>(error: E): Result<never, E> { return { ok: false, error }; } // Usage function parseJSON<T>(json: string): Result<T, SyntaxError> { try { const value = JSON.parse(json) as T; return Ok(value); } catch (error) { return Err(error as SyntaxError); } } // Consuming Result const result = parseJSON<User>(userJson); if (result.ok) { console.log(result.value.name); } else { console.error("Parse failed:", result.error.message); } // Chaining Results function chain<T, U, E>( result: Result<T, E>, fn: (value: T) => Result<U, E>, ): Result<U, E> { return result.ok ? fn(result.value) : result; } ``` **Async Error Handling:** ```typescript // Async/await with proper error handling async function fetchUserOrders(userId: string): Promise<Order[]> { try { const user = await getUser(userId); const orders = await getOrders(user.id); return orders; } catch (error) { if (error instanceof NotFoundError) { return []; // Return empty array for not found } if (error instanceof NetworkError) { // Retry logic return retryFetchOrders(userId); } // Re-throw unexpected errors throw error; } } // Promise error handling function fetchData(url: string): Promise<Data> { return fetch(url) .then((response) => { if (!response.ok) { throw new NetworkError(`HTTP ${response.status}`); } return response.json(); }) .catch((error) => { console.error("Fetch failed:", error); throw error; }); } ``` ### Rust Error Handling **Result and Option Types:** ```rust use std::fs::File; use std::io::{self, Read}; // Result type for operations that can fail fn read_file(path: &str) -> Result<String, io::Error> { let mut file = File::open(path)?; // ? operator propagates errors let mut contents = String::new(); file.read_to_string(&mut contents)?; Ok(contents) } // Custom error types #[derive(Debug)] enum AppError { Io(io::Error), Parse(std::num::ParseIntError), NotFound(String), Validation(String), } impl From<io::Error> for AppError { fn from(error: io::Error) -> Self { AppError::Io(error) } } // Using custom error type fn read_number_from_file(path: &str) -> Result<i32, AppError> { let contents = read_file(path)?; // Auto-converts io::Error let number = contents.trim().parse() .map_err(AppError::Parse)?; // Explicitly convert ParseIntError Ok(number) } // Option for nullable values fn find_user(id: &str) -> Option<User> { users.iter().find(|u| u.id == id).cloned() } // Combining Option and Result fn get_user_age(id: &str) -> Result<u32, AppError> { find_user(id) .ok_or_else(|| AppError::NotFound(id.to_string())) .map(|user| user.age) } ``` ### Go Error Handling **Explicit Error Returns:** ```go // Basic error handling func getUser(id string) (*User, error) { user, err := db.QueryUser(id) if err != nil { return nil, fmt.Errorf("failed to query user: %w", err) } if user == nil { return nil, errors.New("user not found") } return user, nil } // Custom error types type ValidationError struct { Field string Message string } func (e *ValidationError) Error() string { return fmt.Sprintf("validation failed for %s: %s", e.Field, e.Message) } // Sentinel errors for comparison var ( ErrNotFound = errors.New("not found") ErrUnauthorized = errors.New("unauthorized") ErrInvalidInput = errors.New("invalid input") ) // Error checking user, err := getUser("123") if err != nil { if errors.Is(err, ErrNotFound) { // Handle not found } else { // Handle other errors } } // Error wrapping and unwrapping func processUser(id string) error { user, err := getUser(id) if err != nil { return fmt.Errorf("process user failed: %w", err) } // Process user return nil } // Unwrap errors err := processUser("123") if err != nil { var valErr *ValidationError if errors.As(err, &valErr) { fmt.Printf("Validation error: %s\n", valErr.Field) } } ``` ## Universal Patterns ### Pattern 1: Circuit Breaker Prevent cascading failures in distributed systems. ```python from enum import Enum from datetime import datetime, timedelta from typing import Callable, TypeVar T = TypeVar('T') class CircuitState(Enum): CLOSED = "closed" # Normal operation OPEN = "open" # Failing, reject requests HALF_OPEN = "half_open" # Testing if recovered class CircuitBreaker: def __init__( self, failure_threshold: int = 5, timeout: timedelta = timedelta(seconds=60), success_threshold: int = 2 ): self.failure_threshold = failure_threshold self.timeout = timeout self.success_threshold = success_threshold self.failure_count = 0 self.success_count = 0 self.state = CircuitState.CLOSED self.last_failure_time = None def call(self, func: Callable[[], T]) -> T: if self.state == CircuitState.OPEN: if datetime.now() - self.last_failure_time > self.timeout: self.state = CircuitState.HALF_OPEN self.success_count = 0 else: raise Exception("Circuit breaker is OPEN") try: result = func() self.on_success() return result except Exception as e: self.on_failure() raise def on_success(self): self.failure_count = 0 if self.state == CircuitState.HALF_OPEN: self.success_count += 1 if self.success_count >= self.success_threshold: self.state = CircuitState.CLOSED self.success_count = 0 def on_failure(self): self.failure_count += 1 self.last_failure_time = datetime.now() if self.failure_count >= self.failure_threshold: self.state = CircuitState.OPEN # Usage circuit_breaker = CircuitBreaker() def fetch_data(): return circuit_breaker.call(lambda: external_api.get_data()) ``` ### Pattern 2: Error Aggregation Collect multiple errors instead of failing on first error. ```typescript class ErrorCollector { private errors: Error[] = []; add(error: Error): void { this.errors.push(error); } hasErrors(): boolean { return this.errors.length > 0; } getErrors(): Error[] { return [...this.errors]; } throw(): never { if (this.errors.length === 1) { throw this.errors[0]; } throw new AggregateError( this.errors, `${this.errors.length} errors occurred`, ); } } // Usage: Validate multiple fields function validateUser(data: any): User { const errors = new ErrorCollector(); if (!data.email) { errors.add(new ValidationError("Email is required")); } else if (!isValidEmail(data.email)) { errors.add(new ValidationError("Email is invalid")); } if (!data.name || data.name.length < 2) { errors.add(new ValidationError("Name must be at least 2 characters")); } if (!data.age || data.age < 18) { errors.add(new ValidationError("Age must be 18 or older")); } if (errors.hasErrors()) { errors.throw(); } return data as User; } ``` ### Pattern 3: Graceful Degradation Provide fallback functionality when errors occur. ```python from typing import Optional, Callable, TypeVar T = TypeVar('T') def with_fallback( primary: Callable[[], T], fallback: Callable[[], T], log_error: bool = True ) -> T: """Try primary function, fall back to fallback on error.""" try: return primary() except Exception as e: if log_error: logger.error(f"Primary function failed: {e}") return fallback() # Usage def get_user_profile(user_id: str) -> UserProfile: return with_fallback( primary=lambda: fetch_from_cache(user_id), fallback=lambda: fetch_from_database(user_id) ) # Multiple fallbacks def get_exchange_rate(currency: str) -> float: return ( try_function(lambda: api_provider_1.get_rate(currency)) or try_function(lambda: api_provider_2.get_rate(currency)) or try_function(lambda: cache.get_rate(currency)) or DEFAULT_RATE ) def try_function(func: Callable[[], Optional[T]]) -> Optional[T]: try: return func() except Exception: return None ``` ## Best Practices 1. **Fail Fast**: Validate input early, fail quickly 2. **Preserve Context**: Include stack traces, metadata, timestamps 3. **Meaningful Messages**: Explain what happened and how to fix it 4. **Log Appropriately**: Error = log, expected failure = don't spam logs 5. **Handle at Right Level**: Catch where you can meaningfully handle 6. **Clean Up Resources**: Use try-finally, context managers, defer 7. **Don't Swallow Errors**: Log or re-throw, don't silently ignore 8. **Type-Safe Errors**: Use typed errors when possible ```python # Good error handling example def process_order(order_id: str) -> Order: """Process order with comprehensive error handling.""" try: # Validate input if not order_id: raise ValidationError("Order ID is required") # Fetch order order = db.get_order(order_id) if not order: raise NotFoundError("Order", order_id) # Process payment try: payment_result = payment_service.charge(order.total) except PaymentServiceError as e: # Log and wrap external service error logger.error(f"Payment failed for order {order_id}: {e}") raise ExternalServiceError( f"Payment processing failed", service="payment_service", details={"order_id": order_id, "amount": order.total} ) from e # Update order order.status = "completed" order.payment_id = payment_result.id db.save(order) return order except ApplicationError: # Re-raise known application errors raise except Exception as e: # Log unexpected errors logger.exception(f"Unexpected error processing order {order_id}") raise ApplicationError( "Order processing failed", code="INTERNAL_ERROR" ) from e ``` ## Common Pitfalls - **Catching Too Broadly**: `except Exception` hides bugs - **Empty Catch Blocks**: Silently swallowing errors - **Logging and Re-throwing**: Creates duplicate log entries - **Not Cleaning Up**: Forgetting to close files, connections - **Poor Error Messages**: "Error occurred" is not helpful - **Returning Error Codes**: Use exceptions or Result types - **Ignoring Async Errors**: Unhandled promise rejections ## Resources - **references/exception-hierarchy-design.md**: Designing error class hierarchies - **references/error-recovery-strategies.md**: Recovery patterns for different scenarios - **references/async-error-handling.md**: Handling errors in concurrent code - **assets/error-handling-checklist.md**: Review checklist for error handling - **assets/error-message-guide.md**: Writing helpful error messages - **scripts/error-analyzer.py**: Analyze error patterns in logs
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

angular-migration

Migrate from AngularJS to Angular using hybrid mode, incremental

coding
⭐1
# Angular Migration Master AngularJS to Angular migration, including hybrid apps, component conversion, dependency injection changes, and routing migration. ## When to Use This Skill - Migrating AngularJS (1.x) applications to Angular (2+) - Running hybrid AngularJS/Angular applications - Converting directives to components - Modernizing dependency injection - Migrating routing systems - Updating to latest Angular versions - Implementing Angular best practices ## Migration Strategies ### 1. Big Bang (Complete Rewrite) - Rewrite entire app in Angular - Parallel development - Switch over at once - **Best for:** Small apps, green field projects ### 2. Incremental (Hybrid Approach) - Run AngularJS and Angular side-by-side - Migrate feature by feature - ngUpgrade for interop - **Best for:** Large apps, continuous delivery ### 3. Vertical Slice - Migrate one feature completely - New features in Angular, maintain old in AngularJS - Gradually replace - **Best for:** Medium apps, distinct features ## Hybrid App Setup ```typescript // main.ts - Bootstrap hybrid app import { platformBrowserDynamic } from "@angular/platform-browser-dynamic"; import { UpgradeModule } from "@angular/upgrade/static"; import { AppModule } from "./app/app.module"; platformBrowserDynamic() .bootstrapModule(AppModule) .then((platformRef) => { const upgrade = platformRef.injector.get(UpgradeModule); // Bootstrap AngularJS upgrade.bootstrap(document.body, ["myAngularJSApp"], { strictDi: true }); }); ``` ```typescript // app.module.ts import { NgModule } from "@angular/core"; import { BrowserModule } from "@angular/platform-browser"; import { UpgradeModule } from "@angular/upgrade/static"; @NgModule({ imports: [BrowserModule, UpgradeModule], }) export class AppModule { constructor(private upgrade: UpgradeModule) {} ngDoBootstrap() { // Bootstrapped manually in main.ts } } ``` ## Component Migration ### AngularJS Controller → Angular Component ```javascript // Before: AngularJS controller angular .module("myApp") .controller("UserController", function ($scope, UserService) { $scope.user = {}; $scope.loadUser = function (id) { UserService.getUser(id).then(function (user) { $scope.user = user; }); }; $scope.saveUser = function () { UserService.saveUser($scope.user); }; }); ``` ```typescript // After: Angular component import { Component, OnInit } from "@angular/core"; import { UserService } from "./user.service"; @Component({ selector: "app-user", template: ` <div> <h2>{{ user.name }}</h2> <button (click)="saveUser()">Save</button> </div> `, }) export class UserComponent implements OnInit { user: any = {}; constructor(private userService: UserService) {} ngOnInit() { this.loadUser(1); } loadUser(id: number) { this.userService.getUser(id).subscribe((user) => { this.user = user; }); } saveUser() { this.userService.saveUser(this.user); } } ``` ### AngularJS Directive → Angular Component ```javascript // Before: AngularJS directive angular.module("myApp").directive("userCard", function () { return { restrict: "E", scope: { user: "=", onDelete: "&", }, template: ` <div class="card"> <h3>{{ user.name }}</h3> <button ng-click="onDelete()">Delete</button> </div> `, }; }); ``` ```typescript // After: Angular component import { Component, Input, Output, EventEmitter } from "@angular/core"; @Component({ selector: "app-user-card", template: ` <div class="card"> <h3>{{ user.name }}</h3> <button (click)="delete.emit()">Delete</button> </div> `, }) export class UserCardComponent { @Input() user: any; @Output() delete = new EventEmitter<void>(); } // Usage: <app-user-card [user]="user" (delete)="handleDelete()"></app-user-card> ``` ## Service Migration ```javascript // Before: AngularJS service angular.module("myApp").factory("UserService", function ($http) { return { getUser: function (id) { return $http.get("/api/users/" + id); }, saveUser: function (user) { return $http.post("/api/users", user); }, }; }); ``` ```typescript // After: Angular service import { Injectable } from "@angular/core"; import { HttpClient } from "@angular/common/http"; import { Observable } from "rxjs"; @Injectable({ providedIn: "root", }) export class UserService { constructor(private http: HttpClient) {} getUser(id: number): Observable<any> { return this.http.get(`/api/users/${id}`); } saveUser(user: any): Observable<any> { return this.http.post("/api/users", user); } } ``` ## Dependency Injection Changes ### Downgrading Angular → AngularJS ```typescript // Angular service import { Injectable } from "@angular/core"; @Injectable({ providedIn: "root" }) export class NewService { getData() { return "data from Angular"; } } // Make available to AngularJS import { downgradeInjectable } from "@angular/upgrade/static"; angular.module("myApp").factory("newService", downgradeInjectable(NewService)); // Use in AngularJS angular.module("myApp").controller("OldController", function (newService) { console.log(newService.getData()); }); ``` ### Upgrading AngularJS → Angular ```typescript // AngularJS service angular.module('myApp').factory('oldService', function() { return { getData: function() { return 'data from AngularJS'; } }; }); // Make available to Angular import { InjectionToken } from '@angular/core'; export const OLD_SERVICE = new InjectionToken<any>('oldService'); @NgModule({ providers: [ { provide: OLD_SERVICE, useFactory: (i: any) => i.get('oldService'), deps: ['$injector'] } ] }) // Use in Angular @Component({...}) export class NewComponent { constructor(@Inject(OLD_SERVICE) private oldService: any) { console.log(this.oldService.getData()); } } ``` ## Routing Migration ```javascript // Before: AngularJS routing angular.module("myApp").config(function ($routeProvider) { $routeProvider .when("/users", { template: "<user-list></user-list>", }) .when("/users/:id", { template: "<user-detail></user-detail>", }); }); ``` ```typescript // After: Angular routing import { NgModule } from "@angular/core"; import { RouterModule, Routes } from "@angular/router"; const routes: Routes = [ { path: "users", component: UserListComponent }, { path: "users/:id", component: UserDetailComponent }, ]; @NgModule({ imports: [RouterModule.forRoot(routes)], exports: [RouterModule], }) export class AppRoutingModule {} ``` ## Forms Migration ```html <!-- Before: AngularJS --> <form name="userForm" ng-submit="saveUser()"> <input type="text" ng-model="user.name" required /> <input type="email" ng-model="user.email" required /> <button ng-disabled="userForm.$invalid">Save</button> </form> ``` ```typescript // After: Angular (Template-driven) @Component({ template: ` <form #userForm="ngForm" (ngSubmit)="saveUser()"> <input type="text" [(ngModel)]="user.name" name="name" required> <input type="email" [(ngModel)]="user.email" name="email" required> <button [disabled]="userForm.invalid">Save</button> </form> ` }) // Or Reactive Forms (preferred) import { FormBuilder, FormGroup, Validators } from '@angular/forms'; @Component({ template: ` <form [formGroup]="userForm" (ngSubmit)="saveUser()"> <input formControlName="name"> <input formControlName="email"> <button [disabled]="userForm.invalid">Save</button> </form> ` }) export class UserFormComponent { userForm: FormGroup; constructor(private fb: FormBuilder) { this.userForm = this.fb.group({ name: ['', Validators.required], email: ['', [Validators.required, Validators.email]] }); } saveUser() { console.log(this.userForm.value); } } ``` ## Migration Timeline ``` Phase 1: Setup (1-2 weeks) - Install Angular CLI - Set up hybrid app - Configure build tools - Set up testing Phase 2: Infrastructure (2-4 weeks) - Migrate services - Migrate utilities - Set up routing - Migrate shared components Phase 3: Feature Migration (varies) - Migrate feature by feature - Test thoroughly - Deploy incrementally Phase 4: Cleanup (1-2 weeks) - Remove AngularJS code - Remove ngUpgrade - Optimize bundle - Final testing ``` ## Resources - **references/hybrid-mode.md**: Hybrid app patterns - **references/component-migration.md**: Component conversion guide - **references/dependency-injection.md**: DI migration strategies - **references/routing.md**: Routing migration - **assets/hybrid-bootstrap.ts**: Hybrid app template - **assets/migration-timeline.md**: Project planning - **scripts/analyze-angular-app.sh**: App analysis script ## Best Practices 1. **Start with Services**: Migrate services first (easier) 2. **Incremental Approach**: Feature-by-feature migration 3. **Test Continuously**: Test at every step 4. **Use TypeScript**: Migrate to TypeScript early 5. **Follow Style Guide**: Angular style guide from day 1 6. **Optimize Later**: Get it working, then optimize 7. **Document**: Keep migration notes ## Common Pitfalls - Not setting up hybrid app correctly - Migrating UI before logic - Ignoring change detection differences - Not handling scope properly - Mixing patterns (AngularJS + Angular) - Inadequate testing
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

react-modernization

Upgrade React applications to latest versions, migrate from class

coding
⭐1
# React Modernization Master React version upgrades, class to hooks migration, concurrent features adoption, and codemods for automated transformation. ## When to Use This Skill - Upgrading React applications to latest versions - Migrating class components to functional components with hooks - Adopting concurrent React features (Suspense, transitions) - Applying codemods for automated refactoring - Modernizing state management patterns - Updating to TypeScript - Improving performance with React 18+ features ## Version Upgrade Path ### React 16 → 17 → 18 **Breaking Changes by Version:** **React 17:** - Event delegation changes - No event pooling - Effect cleanup timing - JSX transform (no React import needed) **React 18:** - Automatic batching - Concurrent rendering - Strict Mode changes (double invocation) - New root API - Suspense on server ## Class to Hooks Migration ### State Management ```javascript // Before: Class component class Counter extends React.Component { constructor(props) { super(props); this.state = { count: 0, name: "", }; } increment = () => { this.setState({ count: this.state.count + 1 }); }; render() { return ( <div> <p>Count: {this.state.count}</p> <button onClick={this.increment}>Increment</button> </div> ); } } // After: Functional component with hooks function Counter() { const [count, setCount] = useState(0); const [name, setName] = useState(""); const increment = () => { setCount(count + 1); }; return ( <div> <p>Count: {count}</p> <button onClick={increment}>Increment</button> </div> ); } ``` ### Lifecycle Methods to Hooks ```javascript // Before: Lifecycle methods class DataFetcher extends React.Component { state = { data: null, loading: true }; componentDidMount() { this.fetchData(); } componentDidUpdate(prevProps) { if (prevProps.id !== this.props.id) { this.fetchData(); } } componentWillUnmount() { this.cancelRequest(); } fetchData = async () => { const data = await fetch(`/api/${this.props.id}`); this.setState({ data, loading: false }); }; cancelRequest = () => { // Cleanup }; render() { if (this.state.loading) return <div>Loading...</div>; return <div>{this.state.data}</div>; } } // After: useEffect hook function DataFetcher({ id }) { const [data, setData] = useState(null); const [loading, setLoading] = useState(true); useEffect(() => { let cancelled = false; const fetchData = async () => { try { const response = await fetch(`/api/${id}`); const result = await response.json(); if (!cancelled) { setData(result); setLoading(false); } } catch (error) { if (!cancelled) { console.error(error); } } }; fetchData(); // Cleanup function return () => { cancelled = true; }; }, [id]); // Re-run when id changes if (loading) return <div>Loading...</div>; return <div>{data}</div>; } ``` ### Context and HOCs to Hooks ```javascript // Before: Context consumer and HOC const ThemeContext = React.createContext(); class ThemedButton extends React.Component { static contextType = ThemeContext; render() { return ( <button style={{ background: this.context.theme }}> {this.props.children} </button> ); } } // After: useContext hook function ThemedButton({ children }) { const { theme } = useContext(ThemeContext); return <button style={{ background: theme }}>{children}</button>; } // Before: HOC for data fetching function withUser(Component) { return class extends React.Component { state = { user: null }; componentDidMount() { fetchUser().then((user) => this.setState({ user })); } render() { return <Component {...this.props} user={this.state.user} />; } }; } // After: Custom hook function useUser() { const [user, setUser] = useState(null); useEffect(() => { fetchUser().then(setUser); }, []); return user; } function UserProfile() { const user = useUser(); if (!user) return <div>Loading...</div>; return <div>{user.name}</div>; } ``` ## React 18 Concurrent Features ### New Root API ```javascript // Before: React 17 import ReactDOM from "react-dom"; ReactDOM.render(<App />, document.getElementById("root")); // After: React 18 import { createRoot } from "react-dom/client"; const root = createRoot(document.getElementById("root")); root.render(<App />); ``` ### Automatic Batching ```javascript // React 18: All updates are batched function handleClick() { setCount((c) => c + 1); setFlag((f) => !f); // Only one re-render (batched) } // Even in async: setTimeout(() => { setCount((c) => c + 1); setFlag((f) => !f); // Still batched in React 18! }, 1000); // Opt out if needed import { flushSync } from "react-dom"; flushSync(() => { setCount((c) => c + 1); }); // Re-render happens here setFlag((f) => !f); // Another re-render ``` ### Transitions ```javascript import { useState, useTransition } from "react"; function SearchResults() { const [query, setQuery] = useState(""); const [results, setResults] = useState([]); const [isPending, startTransition] = useTransition(); const handleChange = (e) => { // Urgent: Update input immediately setQuery(e.target.value); // Non-urgent: Update results (can be interrupted) startTransition(() => { setResults(searchResults(e.target.value)); }); }; return ( <> <input value={query} onChange={handleChange} /> {isPending && <Spinner />} <Results data={results} /> </> ); } ``` ### Suspense for Data Fetching ```javascript import { Suspense } from "react"; // Resource-based data fetching (with React 18) const resource = fetchProfileData(); function ProfilePage() { return ( <Suspense fallback={<Loading />}> <ProfileDetails /> <Suspense fallback={<Loading />}> <ProfileTimeline /> </Suspense> </Suspense> ); } function ProfileDetails() { // This will suspend if data not ready const user = resource.user.read(); return <h1>{user.name}</h1>; } function ProfileTimeline() { const posts = resource.posts.read(); return <Timeline posts={posts} />; } ``` ## Codemods for Automation ### Run React Codemods ```bash # Rename unsafe lifecycle methods npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/rename-unsafe-lifecycles.js src/ # Update React imports (React 17+) npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/update-react-imports.js src/ # Add error boundaries npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/error-boundaries.js src/ # For TypeScript files npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/rename-unsafe-lifecycles.js --parser=tsx src/ # Dry run to preview changes npx jscodeshift -t https://raw.githubusercontent.com/reactjs/react-codemod/master/transforms/rename-unsafe-lifecycles.js --dry --print src/ # Class to Hooks (third-party) npx codemod react/hooks/convert-class-to-function src/ ``` ### Custom Codemod Example ```javascript // custom-codemod.js module.exports = function (file, api) { const j = api.jscodeshift; const root = j(file.source); // Find setState calls root .find(j.CallExpression, { callee: { type: "MemberExpression", property: { name: "setState" }, }, }) .forEach((path) => { // Transform to useState // ... transformation logic }); return root.toSource(); }; // Run: jscodeshift -t custom-codemod.js src/ ``` ## Performance Optimization ### useMemo and useCallback ```javascript function ExpensiveComponent({ items, filter }) { // Memoize expensive calculation const filteredItems = useMemo(() => { return items.filter((item) => item.category === filter); }, [items, filter]); // Memoize callback to prevent child re-renders const handleClick = useCallback((id) => { console.log("Clicked:", id); }, []); // No dependencies, never changes return <List items={filteredItems} onClick={handleClick} />; } // Child component with memo const List = React.memo(({ items, onClick }) => { return items.map((item) => ( <Item key={item.id} item={item} onClick={onClick} /> )); }); ``` ### Code Splitting ```javascript import { lazy, Suspense } from "react"; // Lazy load components const Dashboard = lazy(() => import("./Dashboard")); const Settings = lazy(() => import("./Settings")); function App() { return ( <Suspense fallback={<Loading />}> <Routes> <Route path="/dashboard" element={<Dashboard />} /> <Route path="/settings" element={<Settings />} /> </Routes> </Suspense> ); } ``` ## TypeScript Migration ```typescript // Before: JavaScript function Button({ onClick, children }) { return <button onClick={onClick}>{children}</button>; } // After: TypeScript interface ButtonProps { onClick: () => void; children: React.ReactNode; } function Button({ onClick, children }: ButtonProps) { return <button onClick={onClick}>{children}</button>; } // Generic components interface ListProps<T> { items: T[]; renderItem: (item: T) => React.ReactNode; } function List<T>({ items, renderItem }: ListProps<T>) { return <>{items.map(renderItem)}</>; } ``` ## Migration Checklist ```markdown ### Pre-Migration - [ ] Update dependencies incrementally (not all at once) - [ ] Review breaking changes in release notes - [ ] Set up testing suite - [ ] Create feature branch ### Class → Hooks Migration - [ ] Identify class components to migrate - [ ] Start with leaf components (no children) - [ ] Convert state to useState - [ ] Convert lifecycle to useEffect - [ ] Convert context to useContext - [ ] Extract custom hooks - [ ] Test thoroughly ### React 18 Upgrade - [ ] Update to React 17 first (if needed) - [ ] Update react and react-dom to 18 - [ ] Update @types/react if using TypeScript - [ ] Change to createRoot API - [ ] Test with StrictMode (double invocation) - [ ] Address concurrent rendering issues - [ ] Adopt Suspense/Transitions where beneficial ### Performance - [ ] Identify performance bottlenecks - [ ] Add React.memo where appropriate - [ ] Use useMemo/useCallback for expensive operations - [ ] Implement code splitting - [ ] Optimize re-renders ### Testing - [ ] Update test utilities (React Testing Library) - [ ] Test with React 18 features - [ ] Check for warnings in console - [ ] Performance testing ``` ## Resources - **references/breaking-changes.md**: Version-specific breaking changes - **references/codemods.md**: Codemod usage guide - **references/hooks-migration.md**: Comprehensive hooks patterns - **references/concurrent-features.md**: React 18 concurrent features - **assets/codemod-config.json**: Codemod configurations - **assets/migration-checklist.md**: Step-by-step checklist - **scripts/apply-codemods.sh**: Automated codemod script ## Best Practices 1. **Incremental Migration**: Don't migrate everything at once 2. **Test Thoroughly**: Comprehensive testing at each step 3. **Use Codemods**: Automate repetitive transformations 4. **Start Simple**: Begin with leaf components 5. **Leverage StrictMode**: Catch issues early 6. **Monitor Performance**: Measure before and after 7. **Document Changes**: Keep migration log ## Common Pitfalls - Forgetting useEffect dependencies - Over-using useMemo/useCallback - Not handling cleanup in useEffect - Mixing class and functional patterns - Ignoring StrictMode warnings - Breaking change assumptions
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

javascript-testing-patterns

Implement comprehensive testing strategies using Jest, Vitest, and

coding
⭐1
# JavaScript Testing Patterns Comprehensive guide for implementing robust testing strategies in JavaScript/TypeScript applications using modern testing frameworks and best practices. ## When to Use This Skill - Setting up test infrastructure for new projects - Writing unit tests for functions and classes - Creating integration tests for APIs and services - Implementing end-to-end tests for user flows - Mocking external dependencies and APIs - Testing React, Vue, or other frontend components - Implementing test-driven development (TDD) - Setting up continuous testing in CI/CD pipelines ## Testing Frameworks ### Jest - Full-Featured Testing Framework **Setup:** ```typescript // jest.config.ts import type { Config } from "jest"; const config: Config = { preset: "ts-jest", testEnvironment: "node", roots: ["<rootDir>/src"], testMatch: ["**/__tests__/**/*.ts", "**/?(*.)+(spec|test).ts"], collectCoverageFrom: [ "src/**/*.ts", "!src/**/*.d.ts", "!src/**/*.interface.ts", ], coverageThreshold: { global: { branches: 80, functions: 80, lines: 80, statements: 80, }, }, setupFilesAfterEnv: ["<rootDir>/src/test/setup.ts"], }; export default config; ``` ### Vitest - Fast, Vite-Native Testing **Setup:** ```typescript // vitest.config.ts import { defineConfig } from "vitest/config"; export default defineConfig({ test: { globals: true, environment: "node", coverage: { provider: "v8", reporter: ["text", "json", "html"], exclude: ["**/*.d.ts", "**/*.config.ts", "**/dist/**"], }, setupFiles: ["./src/test/setup.ts"], }, }); ``` ## Unit Testing Patterns ### Pattern 1: Testing Pure Functions ```typescript // utils/calculator.ts export function add(a: number, b: number): number { return a + b; } export function divide(a: number, b: number): number { if (b === 0) { throw new Error("Division by zero"); } return a / b; } // utils/calculator.test.ts import { describe, it, expect } from "vitest"; import { add, divide } from "./calculator"; describe("Calculator", () => { describe("add", () => { it("should add two positive numbers", () => { expect(add(2, 3)).toBe(5); }); it("should add negative numbers", () => { expect(add(-2, -3)).toBe(-5); }); it("should handle zero", () => { expect(add(0, 5)).toBe(5); expect(add(5, 0)).toBe(5); }); }); describe("divide", () => { it("should divide two numbers", () => { expect(divide(10, 2)).toBe(5); }); it("should handle decimal results", () => { expect(divide(5, 2)).toBe(2.5); }); it("should throw error when dividing by zero", () => { expect(() => divide(10, 0)).toThrow("Division by zero"); }); }); }); ``` ### Pattern 2: Testing Classes ```typescript // services/user.service.ts export class UserService { private users: Map<string, User> = new Map(); create(user: User): User { if (this.users.has(user.id)) { throw new Error("User already exists"); } this.users.set(user.id, user); return user; } findById(id: string): User | undefined { return this.users.get(id); } update(id: string, updates: Partial<User>): User { const user = this.users.get(id); if (!user) { throw new Error("User not found"); } const updated = { ...user, ...updates }; this.users.set(id, updated); return updated; } delete(id: string): boolean { return this.users.delete(id); } } // services/user.service.test.ts import { describe, it, expect, beforeEach } from "vitest"; import { UserService } from "./user.service"; describe("UserService", () => { let service: UserService; beforeEach(() => { service = new UserService(); }); describe("create", () => { it("should create a new user", () => { const user = { id: "1", name: "John", email: "john@example.com" }; const created = service.create(user); expect(created).toEqual(user); expect(service.findById("1")).toEqual(user); }); it("should throw error if user already exists", () => { const user = { id: "1", name: "John", email: "john@example.com" }; service.create(user); expect(() => service.create(user)).toThrow("User already exists"); }); }); describe("update", () => { it("should update existing user", () => { const user = { id: "1", name: "John", email: "john@example.com" }; service.create(user); const updated = service.update("1", { name: "Jane" }); expect(updated.name).toBe("Jane"); expect(updated.email).toBe("john@example.com"); }); it("should throw error if user not found", () => { expect(() => service.update("999", { name: "Jane" })).toThrow( "User not found", ); }); }); }); ``` ### Pattern 3: Testing Async Functions ```typescript // services/api.service.ts export class ApiService { async fetchUser(id: string): Promise<User> { const response = await fetch(`https://api.example.com/users/${id}`); if (!response.ok) { throw new Error("User not found"); } return response.json(); } async createUser(user: CreateUserDTO): Promise<User> { const response = await fetch("https://api.example.com/users", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(user), }); return response.json(); } } // services/api.service.test.ts import { describe, it, expect, vi, beforeEach } from "vitest"; import { ApiService } from "./api.service"; // Mock fetch globally global.fetch = vi.fn(); describe("ApiService", () => { let service: ApiService; beforeEach(() => { service = new ApiService(); vi.clearAllMocks(); }); describe("fetchUser", () => { it("should fetch user successfully", async () => { const mockUser = { id: "1", name: "John", email: "john@example.com" }; (fetch as any).mockResolvedValueOnce({ ok: true, json: async () => mockUser, }); const user = await service.fetchUser("1"); expect(user).toEqual(mockUser); expect(fetch).toHaveBeenCalledWith("https://api.example.com/users/1"); }); it("should throw error if user not found", async () => { (fetch as any).mockResolvedValueOnce({ ok: false, }); await expect(service.fetchUser("999")).rejects.toThrow("User not found"); }); }); describe("createUser", () => { it("should create user successfully", async () => { const newUser = { name: "John", email: "john@example.com" }; const createdUser = { id: "1", ...newUser }; (fetch as any).mockResolvedValueOnce({ ok: true, json: async () => createdUser, }); const user = await service.createUser(newUser); expect(user).toEqual(createdUser); expect(fetch).toHaveBeenCalledWith( "https://api.example.com/users", expect.objectContaining({ method: "POST", body: JSON.stringify(newUser), }), ); }); }); }); ``` ## Mocking Patterns ### Pattern 1: Mocking Modules ```typescript // services/email.service.ts import nodemailer from "nodemailer"; export class EmailService { private transporter = nodemailer.createTransport({ host: process.env.SMTP_HOST, port: 587, auth: { user: process.env.SMTP_USER, pass: process.env.SMTP_PASS, }, }); async sendEmail(to: string, subject: string, html: string) { await this.transporter.sendMail({ from: process.env.EMAIL_FROM, to, subject, html, }); } } // services/email.service.test.ts import { describe, it, expect, vi, beforeEach } from "vitest"; import { EmailService } from "./email.service"; vi.mock("nodemailer", () => ({ default: { createTransport: vi.fn(() => ({ sendMail: vi.fn().mockResolvedValue({ messageId: "123" }), })), }, })); describe("EmailService", () => { let service: EmailService; beforeEach(() => { service = new EmailService(); }); it("should send email successfully", async () => { await service.sendEmail( "test@example.com", "Test Subject", "<p>Test Body</p>", ); expect(service["transporter"].sendMail).toHaveBeenCalledWith( expect.objectContaining({ to: "test@example.com", subject: "Test Subject", }), ); }); }); ``` ### Pattern 2: Dependency Injection for Testing ```typescript // services/user.service.ts export interface IUserRepository { findById(id: string): Promise<User | null>; create(user: User): Promise<User>; } export class UserService { constructor(private userRepository: IUserRepository) {} async getUser(id: string): Promise<User> { const user = await this.userRepository.findById(id); if (!user) { throw new Error("User not found"); } return user; } async createUser(userData: CreateUserDTO): Promise<User> { // Business logic here const user = { id: generateId(), ...userData }; return this.userRepository.create(user); } } // services/user.service.test.ts import { describe, it, expect, vi, beforeEach } from "vitest"; import { UserService, IUserRepository } from "./user.service"; describe("UserService", () => { let service: UserService; let mockRepository: IUserRepository; beforeEach(() => { mockRepository = { findById: vi.fn(), create: vi.fn(), }; service = new UserService(mockRepository); }); describe("getUser", () => { it("should return user if found", async () => { const mockUser = { id: "1", name: "John", email: "john@example.com" }; vi.mocked(mockRepository.findById).mockResolvedValue(mockUser); const user = await service.getUser("1"); expect(user).toEqual(mockUser); expect(mockRepository.findById).toHaveBeenCalledWith("1"); }); it("should throw error if user not found", async () => { vi.mocked(mockRepository.findById).mockResolvedValue(null); await expect(service.getUser("999")).rejects.toThrow("User not found"); }); }); describe("createUser", () => { it("should create user successfully", async () => { const userData = { name: "John", email: "john@example.com" }; const createdUser = { id: "1", ...userData }; vi.mocked(mockRepository.create).mockResolvedValue(createdUser); const user = await service.createUser(userData); expect(user).toEqual(createdUser); expect(mockRepository.create).toHaveBeenCalled(); }); }); }); ``` ### Pattern 3: Spying on Functions ```typescript // utils/logger.ts export const logger = { info: (message: string) => console.log(`INFO: ${message}`), error: (message: string) => console.error(`ERROR: ${message}`), }; // services/order.service.ts import { logger } from "../utils/logger"; export class OrderService { async processOrder(orderId: string): Promise<void> { logger.info(`Processing order ${orderId}`); // Process order logic logger.info(`Order ${orderId} processed successfully`); } } // services/order.service.test.ts import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { OrderService } from "./order.service"; import { logger } from "../utils/logger"; describe("OrderService", () => { let service: OrderService; let loggerSpy: any; beforeEach(() => { service = new OrderService(); loggerSpy = vi.spyOn(logger, "info"); }); afterEach(() => { loggerSpy.mockRestore(); }); it("should log order processing", async () => { await service.processOrder("123"); expect(loggerSpy).toHaveBeenCalledWith("Processing order 123"); expect(loggerSpy).toHaveBeenCalledWith("Order 123 processed successfully"); expect(loggerSpy).toHaveBeenCalledTimes(2); }); }); ``` ## Integration Testing ### Pattern 1: API Integration Tests ```typescript // tests/integration/user.api.test.ts import request from "supertest"; import { app } from "../../src/app"; import { pool } from "../../src/config/database"; describe("User API Integration Tests", () => { beforeAll(async () => { // Setup test database await pool.query("CREATE TABLE IF NOT EXISTS users (...)"); }); afterAll(async () => { // Cleanup await pool.query("DROP TABLE IF EXISTS users"); await pool.end(); }); beforeEach(async () => { // Clear data before each test await pool.query("TRUNCATE TABLE users CASCADE"); }); describe("POST /api/users", () => { it("should create a new user", async () => { const userData = { name: "John Doe", email: "john@example.com", password: "password123", }; const response = await request(app) .post("/api/users") .send(userData) .expect(201); expect(response.body).toMatchObject({ name: userData.name, email: userData.email, }); expect(response.body).toHaveProperty("id"); expect(response.body).not.toHaveProperty("password"); }); it("should return 400 if email is invalid", async () => { const userData = { name: "John Doe", email: "invalid-email", password: "password123", }; const response = await request(app) .post("/api/users") .send(userData) .expect(400); expect(response.body).toHaveProperty("error"); }); it("should return 409 if email already exists", async () => { const userData = { name: "John Doe", email: "john@example.com", password: "password123", }; await request(app).post("/api/users").send(userData); const response = await request(app) .post("/api/users") .send(userData) .expect(409); expect(response.body.error).toContain("already exists"); }); }); describe("GET /api/users/:id", () => { it("should get user by id", async () => { const createResponse = await request(app).post("/api/users").send({ name: "John Doe", email: "john@example.com", password: "password123", }); const userId = createResponse.body.id; const response = await request(app) .get(`/api/users/${userId}`) .expect(200); expect(response.body).toMatchObject({ id: userId, name: "John Doe", email: "john@example.com", }); }); it("should return 404 if user not found", async () => { await request(app).get("/api/users/999").expect(404); }); }); describe("Authentication", () => { it("should require authentication for protected routes", async () => { await request(app).get("/api/users/me").expect(401); }); it("should allow access with valid token", async () => { // Create user and login await request(app).post("/api/users").send({ name: "John Doe", email: "john@example.com", password: "password123", }); const loginResponse = await request(app).post("/api/auth/login").send({ email: "john@example.com", password: "password123", }); const token = loginResponse.body.token; const response = await request(app) .get("/api/users/me") .set("Authorization", `Bearer ${token}`) .expect(200); expect(response.body.email).toBe("john@example.com"); }); }); }); ``` ### Pattern 2: Database Integration Tests ```typescript // tests/integration/user.repository.test.ts import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; import { Pool } from "pg"; import { UserRepository } from "../../src/repositories/user.repository"; describe("UserRepository Integration Tests", () => { let pool: Pool; let repository: UserRepository; beforeAll(async () => { pool = new Pool({ host: "localhost", port: 5432, database: "test_db", user: "test_user", password: "test_password", }); repository = new UserRepository(pool); // Create tables await pool.query(` CREATE TABLE IF NOT EXISTS users ( id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) UNIQUE NOT NULL, password VARCHAR(255) NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) `); }); afterAll(async () => { await pool.query("DROP TABLE IF EXISTS users"); await pool.end(); }); beforeEach(async () => { await pool.query("TRUNCATE TABLE users CASCADE"); }); it("should create a user", async () => { const user = await repository.create({ name: "John Doe", email: "john@example.com", password: "hashed_password", }); expect(user).toHaveProperty("id"); expect(user.name).toBe("John Doe"); expect(user.email).toBe("john@example.com"); }); it("should find user by email", async () => { await repository.create({ name: "John Doe", email: "john@example.com", password: "hashed_password", }); const user = await repository.findByEmail("john@example.com"); expect(user).toBeTruthy(); expect(user?.name).toBe("John Doe"); }); it("should return null if user not found", async () => { const user = await repository.findByEmail("nonexistent@example.com"); expect(user).toBeNull(); }); }); ``` ## Frontend Testing with Testing Library ### Pattern 1: React Component Testing ```typescript // components/UserForm.tsx import { useState } from 'react'; interface Props { onSubmit: (user: { name: string; email: string }) => void; } export function UserForm({ onSubmit }: Props) { const [name, setName] = useState(''); const [email, setEmail] = useState(''); const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); onSubmit({ name, email }); }; return ( <form onSubmit={handleSubmit}> <input type="text" placeholder="Name" value={name} onChange={(e) => setName(e.target.value)} data-testid="name-input" /> <input type="email" placeholder="Email" value={email} onChange={(e) => setEmail(e.target.value)} data-testid="email-input" /> <button type="submit">Submit</button> </form> ); } // components/UserForm.test.tsx import { render, screen, fireEvent } from '@testing-library/react'; import { describe, it, expect, vi } from 'vitest'; import { UserForm } from './UserForm'; describe('UserForm', () => { it('should render form inputs', () => { render(<UserForm onSubmit={vi.fn()} />); expect(screen.getByPlaceholderText('Name')).toBeInTheDocument(); expect(screen.getByPlaceholderText('Email')).toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Submit' })).toBeInTheDocument(); }); it('should update input values', () => { render(<UserForm onSubmit={vi.fn()} />); const nameInput = screen.getByTestId('name-input') as HTMLInputElement; const emailInput = screen.getByTestId('email-input') as HTMLInputElement; fireEvent.change(nameInput, { target: { value: 'John Doe' } }); fireEvent.change(emailInpu
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

modern-javascript-patterns

Master ES6+ features including async/await, destructuring, spread

coding
⭐1
# Modern JavaScript Patterns Comprehensive guide for mastering modern JavaScript (ES6+) features, functional programming patterns, and best practices for writing clean, maintainable, and performant code. ## When to Use This Skill - Refactoring legacy JavaScript to modern syntax - Implementing functional programming patterns - Optimizing JavaScript performance - Writing maintainable and readable code - Working with asynchronous operations - Building modern web applications - Migrating from callbacks to Promises/async-await - Implementing data transformation pipelines ## ES6+ Core Features ### 1. Arrow Functions **Syntax and Use Cases:** ```javascript // Traditional function function add(a, b) { return a + b; } // Arrow function const add = (a, b) => a + b; // Single parameter (parentheses optional) const double = (x) => x * 2; // No parameters const getRandom = () => Math.random(); // Multiple statements (need curly braces) const processUser = (user) => { const normalized = user.name.toLowerCase(); return { ...user, name: normalized }; }; // Returning objects (wrap in parentheses) const createUser = (name, age) => ({ name, age }); ``` **Lexical 'this' Binding:** ```javascript class Counter { constructor() { this.count = 0; } // Arrow function preserves 'this' context increment = () => { this.count++; }; // Traditional function loses 'this' in callbacks incrementTraditional() { setTimeout(function () { this.count++; // 'this' is undefined }, 1000); } // Arrow function maintains 'this' incrementArrow() { setTimeout(() => { this.count++; // 'this' refers to Counter instance }, 1000); } } ``` ### 2. Destructuring **Object Destructuring:** ```javascript const user = { id: 1, name: "John Doe", email: "john@example.com", address: { city: "New York", country: "USA", }, }; // Basic destructuring const { name, email } = user; // Rename variables const { name: userName, email: userEmail } = user; // Default values const { age = 25 } = user; // Nested destructuring const { address: { city, country }, } = user; // Rest operator const { id, ...userWithoutId } = user; // Function parameters function greet({ name, age = 18 }) { console.log(`Hello ${name}, you are ${age}`); } greet(user); ``` **Array Destructuring:** ```javascript const numbers = [1, 2, 3, 4, 5]; // Basic destructuring const [first, second] = numbers; // Skip elements const [, , third] = numbers; // Rest operator const [head, ...tail] = numbers; // Swapping variables let a = 1, b = 2; [a, b] = [b, a]; // Function return values function getCoordinates() { return [10, 20]; } const [x, y] = getCoordinates(); // Default values const [one, two, three = 0] = [1, 2]; ``` ### 3. Spread and Rest Operators **Spread Operator:** ```javascript // Array spreading const arr1 = [1, 2, 3]; const arr2 = [4, 5, 6]; const combined = [...arr1, ...arr2]; // Object spreading const defaults = { theme: "dark", lang: "en" }; const userPrefs = { theme: "light" }; const settings = { ...defaults, ...userPrefs }; // Function arguments const numbers = [1, 2, 3]; Math.max(...numbers); // Copying arrays/objects (shallow copy) const copy = [...arr1]; const objCopy = { ...user }; // Adding items immutably const newArr = [...arr1, 4, 5]; const newObj = { ...user, age: 30 }; ``` **Rest Parameters:** ```javascript // Collect function arguments function sum(...numbers) { return numbers.reduce((total, num) => total + num, 0); } sum(1, 2, 3, 4, 5); // With regular parameters function greet(greeting, ...names) { return `${greeting} ${names.join(", ")}`; } greet("Hello", "John", "Jane", "Bob"); // Object rest const { id, ...userData } = user; // Array rest const [first, ...rest] = [1, 2, 3, 4, 5]; ``` ### 4. Template Literals ```javascript // Basic usage const name = "John"; const greeting = `Hello, ${name}!`; // Multi-line strings const html = ` <div> <h1>${title}</h1> <p>${content}</p> </div> `; // Expression evaluation const price = 19.99; const total = `Total: $${(price * 1.2).toFixed(2)}`; // Tagged template literals function highlight(strings, ...values) { return strings.reduce((result, str, i) => { const value = values[i] || ""; return result + str + `<mark>${value}</mark>`; }, ""); } const name = "John"; const age = 30; const html = highlight`Name: ${name}, Age: ${age}`; // Output: "Name: <mark>John</mark>, Age: <mark>30</mark>" ``` ### 5. Enhanced Object Literals ```javascript const name = "John"; const age = 30; // Shorthand property names const user = { name, age }; // Shorthand method names const calculator = { add(a, b) { return a + b; }, subtract(a, b) { return a - b; }, }; // Computed property names const field = "email"; const user = { name: "John", [field]: "john@example.com", [`get${field.charAt(0).toUpperCase()}${field.slice(1)}`]() { return this[field]; }, }; // Dynamic property creation const createUser = (name, ...props) => { return props.reduce( (user, [key, value]) => ({ ...user, [key]: value, }), { name }, ); }; const user = createUser("John", ["age", 30], ["email", "john@example.com"]); ``` ## Asynchronous Patterns ### 1. Promises **Creating and Using Promises:** ```javascript // Creating a promise const fetchUser = (id) => { return new Promise((resolve, reject) => { setTimeout(() => { if (id > 0) { resolve({ id, name: "John" }); } else { reject(new Error("Invalid ID")); } }, 1000); }); }; // Using promises fetchUser(1) .then((user) => console.log(user)) .catch((error) => console.error(error)) .finally(() => console.log("Done")); // Chaining promises fetchUser(1) .then((user) => fetchUserPosts(user.id)) .then((posts) => processPosts(posts)) .then((result) => console.log(result)) .catch((error) => console.error(error)); ``` **Promise Combinators:** ```javascript // Promise.all - Wait for all promises const promises = [fetchUser(1), fetchUser(2), fetchUser(3)]; Promise.all(promises) .then((users) => console.log(users)) .catch((error) => console.error("At least one failed:", error)); // Promise.allSettled - Wait for all, regardless of outcome Promise.allSettled(promises).then((results) => { results.forEach((result) => { if (result.status === "fulfilled") { console.log("Success:", result.value); } else { console.log("Error:", result.reason); } }); }); // Promise.race - First to complete Promise.race(promises) .then((winner) => console.log("First:", winner)) .catch((error) => console.error(error)); // Promise.any - First to succeed Promise.any(promises) .then((first) => console.log("First success:", first)) .catch((error) => console.error("All failed:", error)); ``` ### 2. Async/Await **Basic Usage:** ```javascript // Async function always returns a Promise async function fetchUser(id) { const response = await fetch(`/api/users/${id}`); const user = await response.json(); return user; } // Error handling with try/catch async function getUserData(id) { try { const user = await fetchUser(id); const posts = await fetchUserPosts(user.id); return { user, posts }; } catch (error) { console.error("Error fetching data:", error); throw error; } } // Sequential vs Parallel execution async function sequential() { const user1 = await fetchUser(1); // Wait const user2 = await fetchUser(2); // Then wait return [user1, user2]; } async function parallel() { const [user1, user2] = await Promise.all([fetchUser(1), fetchUser(2)]); return [user1, user2]; } ``` **Advanced Patterns:** ```javascript // Async IIFE (async () => { const result = await someAsyncOperation(); console.log(result); })(); // Async iteration async function processUsers(userIds) { for (const id of userIds) { const user = await fetchUser(id); await processUser(user); } } // Top-level await (ES2022) const config = await fetch("/config.json").then((r) => r.json()); // Retry logic async function fetchWithRetry(url, retries = 3) { for (let i = 0; i < retries; i++) { try { return await fetch(url); } catch (error) { if (i === retries - 1) throw error; await new Promise((resolve) => setTimeout(resolve, 1000 * (i + 1))); } } } // Timeout wrapper async function withTimeout(promise, ms) { const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), ms), ); return Promise.race([promise, timeout]); } ``` ## Functional Programming Patterns ### 1. Array Methods **Map, Filter, Reduce:** ```javascript const users = [ { id: 1, name: "John", age: 30, active: true }, { id: 2, name: "Jane", age: 25, active: false }, { id: 3, name: "Bob", age: 35, active: true }, ]; // Map - Transform array const names = users.map((user) => user.name); const upperNames = users.map((user) => user.name.toUpperCase()); // Filter - Select elements const activeUsers = users.filter((user) => user.active); const adults = users.filter((user) => user.age >= 18); // Reduce - Aggregate data const totalAge = users.reduce((sum, user) => sum + user.age, 0); const avgAge = totalAge / users.length; // Group by property const byActive = users.reduce((groups, user) => { const key = user.active ? "active" : "inactive"; return { ...groups, [key]: [...(groups[key] || []), user], }; }, {}); // Chaining methods const result = users .filter((user) => user.active) .map((user) => user.name) .sort() .join(", "); ``` **Advanced Array Methods:** ```javascript // Find - First matching element const user = users.find((u) => u.id === 2); // FindIndex - Index of first match const index = users.findIndex((u) => u.name === "Jane"); // Some - At least one matches const hasActive = users.some((u) => u.active); // Every - All match const allAdults = users.every((u) => u.age >= 18); // FlatMap - Map and flatten const userTags = [ { name: "John", tags: ["admin", "user"] }, { name: "Jane", tags: ["user"] }, ]; const allTags = userTags.flatMap((u) => u.tags); // From - Create array from iterable const str = "hello"; const chars = Array.from(str); const numbers = Array.from({ length: 5 }, (_, i) => i + 1); // Of - Create array from arguments const arr = Array.of(1, 2, 3); ``` ### 2. Higher-Order Functions **Functions as Arguments:** ```javascript // Custom forEach function forEach(array, callback) { for (let i = 0; i < array.length; i++) { callback(array[i], i, array); } } // Custom map function map(array, transform) { const result = []; for (const item of array) { result.push(transform(item)); } return result; } // Custom filter function filter(array, predicate) { const result = []; for (const item of array) { if (predicate(item)) { result.push(item); } } return result; } ``` **Functions Returning Functions:** ```javascript // Currying const multiply = (a) => (b) => a * b; const double = multiply(2); const triple = multiply(3); console.log(double(5)); // 10 console.log(triple(5)); // 15 // Partial application function partial(fn, ...args) { return (...moreArgs) => fn(...args, ...moreArgs); } const add = (a, b, c) => a + b + c; const add5 = partial(add, 5); console.log(add5(3, 2)); // 10 // Memoization function memoize(fn) { const cache = new Map(); return (...args) => { const key = JSON.stringify(args); if (cache.has(key)) { return cache.get(key); } const result = fn(...args); cache.set(key, result); return result; }; } const fibonacci = memoize((n) => { if (n <= 1) return n; return fibonacci(n - 1) + fibonacci(n - 2); }); ``` ### 3. Composition and Piping ```javascript // Function composition const compose = (...fns) => (x) => fns.reduceRight((acc, fn) => fn(acc), x); const pipe = (...fns) => (x) => fns.reduce((acc, fn) => fn(acc), x); // Example usage const addOne = (x) => x + 1; const double = (x) => x * 2; const square = (x) => x * x; const composed = compose(square, double, addOne); console.log(composed(3)); // ((3 + 1) * 2)^2 = 64 const piped = pipe(addOne, double, square); console.log(piped(3)); // ((3 + 1) * 2)^2 = 64 // Practical example const processUser = pipe( (user) => ({ ...user, name: user.name.trim() }), (user) => ({ ...user, email: user.email.toLowerCase() }), (user) => ({ ...user, age: parseInt(user.age) }), ); const user = processUser({ name: " John ", email: "JOHN@EXAMPLE.COM", age: "30", }); ``` ### 4. Pure Functions and Immutability ```javascript // Impure function (modifies input) function addItemImpure(cart, item) { cart.items.push(item); cart.total += item.price; return cart; } // Pure function (no side effects) function addItemPure(cart, item) { return { ...cart, items: [...cart.items, item], total: cart.total + item.price, }; } // Immutable array operations const numbers = [1, 2, 3, 4, 5]; // Add to array const withSix = [...numbers, 6]; // Remove from array const withoutThree = numbers.filter((n) => n !== 3); // Update array element const doubled = numbers.map((n) => (n === 3 ? n * 2 : n)); // Immutable object operations const user = { name: "John", age: 30 }; // Update property const olderUser = { ...user, age: 31 }; // Add property const withEmail = { ...user, email: "john@example.com" }; // Remove property const { age, ...withoutAge } = user; // Deep cloning (simple approach) const deepClone = (obj) => JSON.parse(JSON.stringify(obj)); // Better deep cloning const structuredClone = (obj) => globalThis.structuredClone(obj); ``` ## Modern Class Features ```javascript // Class syntax class User { // Private fields #password; // Public fields id; name; // Static field static count = 0; constructor(id, name, password) { this.id = id; this.name = name; this.#password = password; User.count++; } // Public method greet() { return `Hello, ${this.name}`; } // Private method #hashPassword(password) { return `hashed_${password}`; } // Getter get displayName() { return this.name.toUpperCase(); } // Setter set password(newPassword) { this.#password = this.#hashPassword(newPassword); } // Static method static create(id, name, password) { return new User(id, name, password); } } // Inheritance class Admin extends User { constructor(id, name, password, role) { super(id, name, password); this.role = role; } greet() { return `${super.greet()}, I'm an admin`; } } ``` ## Modules (ES6) ```javascript // Exporting // math.js export const PI = 3.14159; export function add(a, b) { return a + b; } export class Calculator { // ... } // Default export export default function multiply(a, b) { return a * b; } // Importing // app.js import multiply, { PI, add, Calculator } from "./math.js"; // Rename imports import { add as sum } from "./math.js"; // Import all import * as Math from "./math.js"; // Dynamic imports const module = await import("./math.js"); const { add } = await import("./math.js"); // Conditional loading if (condition) { const module = await import("./feature.js"); module.init(); } ``` ## Iterators and Generators ```javascript // Custom iterator const range = { from: 1, to: 5, [Symbol.iterator]() { return { current: this.from, last: this.to, next() { if (this.current <= this.last) { return { done: false, value: this.current++ }; } else { return { done: true }; } }, }; }, }; for (const num of range) { console.log(num); // 1, 2, 3, 4, 5 } // Generator function function* rangeGenerator(from, to) { for (let i = from; i <= to; i++) { yield i; } } for (const num of rangeGenerator(1, 5)) { console.log(num); } // Infinite generator function* fibonacci() { let [prev, curr] = [0, 1]; while (true) { yield curr; [prev, curr] = [curr, prev + curr]; } } // Async generator async function* fetchPages(url) { let page = 1; while (true) { const response = await fetch(`${url}?page=${page}`); const data = await response.json(); if (data.length === 0) break; yield data; page++; } } for await (const page of fetchPages("/api/users")) { console.log(page); } ``` ## Modern Operators ```javascript // Optional chaining const user = { name: "John", address: { city: "NYC" } }; const city = user?.address?.city; const zipCode = user?.address?.zipCode; // undefined // Function call const result = obj.method?.(); // Array access const first = arr?.[0]; // Nullish coalescing const value = null ?? "default"; // 'default' const value = undefined ?? "default"; // 'default' const value = 0 ?? "default"; // 0 (not 'default') const value = "" ?? "default"; // '' (not 'default') // Logical assignment let a = null; a ??= "default"; // a = 'default' let b = 5; b ??= 10; // b = 5 (unchanged) let obj = { count: 0 }; obj.count ||= 1; // obj.count = 1 obj.count &&= 2; // obj.count = 2 ``` ## Performance Optimization ```javascript // Debounce function debounce(fn, delay) { let timeoutId; return (...args) => { clearTimeout(timeoutId); timeoutId = setTimeout(() => fn(...args), delay); }; } const searchDebounced = debounce(search, 300); // Throttle function throttle(fn, limit) { let inThrottle; return (...args) => { if (!inThrottle) { fn(...args); inThrottle = true; setTimeout(() => (inThrottle = false), limit); } }; } const scrollThrottled = throttle(handleScroll, 100); // Lazy evaluation function* lazyMap(iterable, transform) { for (const item of iterable) { yield transform(item); } } // Use only what you need const numbers = [1, 2, 3, 4, 5]; const doubled = lazyMap(numbers, (x) => x * 2); const first = doubled.next().value; // Only computes first value ``` ## Best Practices 1. **Use const by default**: Only use let when reassignment is needed 2. **Prefer arrow functions**: Especially for callbacks 3. **Use template literals**: Instead of string concatenation 4. **Destructure objects and arrays**: For cleaner code 5. **Use async/await**: Instead of Promise chains 6. **Avoid mutating data**: Use spread operator and array methods 7. **Use optional chaining**: Prevent "Cannot read property of undefined" 8. **Use nullish coalescing**: For default values 9. **Prefer array methods**: Over traditional loops 10. **Use modules**: For better code organization 11. **Write pure functions**: Easier to test and reason about 12. **Use meaningful variable names**: Self-documenting code 13. **Keep functions small**: Single responsibility principle 14. **Handle errors properly**: Use try/catch with async/await 15. **Use strict mode**: `'use strict'` for better error catching ## Common Pitfalls 1. **this binding confusion**: Use arrow functions or bind() 2. **Async/await without error handling**: Always use try/catch 3. **Promise creation
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

nodejs-backend-patterns

Build production-ready Node.js backend services with

coding
⭐1
# Node.js Backend Patterns Comprehensive guidance for building scalable, maintainable, and production-ready Node.js backend applications with modern frameworks, architectural patterns, and best practices. ## When to Use This Skill - Building REST APIs or GraphQL servers - Creating microservices with Node.js - Implementing authentication and authorization - Designing scalable backend architectures - Setting up middleware and error handling - Integrating databases (SQL and NoSQL) - Building real-time applications with WebSockets - Implementing background job processing ## Core Frameworks ### Express.js - Minimalist Framework **Basic Setup:** ```typescript import express, { Request, Response, NextFunction } from "express"; import helmet from "helmet"; import cors from "cors"; import compression from "compression"; const app = express(); // Security middleware app.use(helmet()); app.use(cors({ origin: process.env.ALLOWED_ORIGINS?.split(",") })); app.use(compression()); // Body parsing app.use(express.json({ limit: "10mb" })); app.use(express.urlencoded({ extended: true, limit: "10mb" })); // Request logging app.use((req: Request, res: Response, next: NextFunction) => { console.log(`${req.method} ${req.path}`); next(); }); const PORT = process.env.PORT || 3000; app.listen(PORT, () => { console.log(`Server running on port ${PORT}`); }); ``` ### Fastify - High Performance Framework **Basic Setup:** ```typescript import Fastify from "fastify"; import helmet from "@fastify/helmet"; import cors from "@fastify/cors"; import compress from "@fastify/compress"; const fastify = Fastify({ logger: { level: process.env.LOG_LEVEL || "info", transport: { target: "pino-pretty", options: { colorize: true }, }, }, }); // Plugins await fastify.register(helmet); await fastify.register(cors, { origin: true }); await fastify.register(compress); // Type-safe routes with schema validation fastify.post<{ Body: { name: string; email: string }; Reply: { id: string; name: string }; }>( "/users", { schema: { body: { type: "object", required: ["name", "email"], properties: { name: { type: "string", minLength: 1 }, email: { type: "string", format: "email" }, }, }, }, }, async (request, reply) => { const { name, email } = request.body; return { id: "123", name }; }, ); await fastify.listen({ port: 3000, host: "0.0.0.0" }); ``` ## Architectural Patterns ### Pattern 1: Layered Architecture **Structure:** ``` src/ ā”œā”€ā”€ controllers/ # Handle HTTP requests/responses ā”œā”€ā”€ services/ # Business logic ā”œā”€ā”€ repositories/ # Data access layer ā”œā”€ā”€ models/ # Data models ā”œā”€ā”€ middleware/ # Express/Fastify middleware ā”œā”€ā”€ routes/ # Route definitions ā”œā”€ā”€ utils/ # Helper functions ā”œā”€ā”€ config/ # Configuration └── types/ # TypeScript types ``` **Controller Layer:** ```typescript // controllers/user.controller.ts import { Request, Response, NextFunction } from "express"; import { UserService } from "../services/user.service"; import { CreateUserDTO, UpdateUserDTO } from "../types/user.types"; export class UserController { constructor(private userService: UserService) {} async createUser(req: Request, res: Response, next: NextFunction) { try { const userData: CreateUserDTO = req.body; const user = await this.userService.createUser(userData); res.status(201).json(user); } catch (error) { next(error); } } async getUser(req: Request, res: Response, next: NextFunction) { try { const { id } = req.params; const user = await this.userService.getUserById(id); res.json(user); } catch (error) { next(error); } } async updateUser(req: Request, res: Response, next: NextFunction) { try { const { id } = req.params; const updates: UpdateUserDTO = req.body; const user = await this.userService.updateUser(id, updates); res.json(user); } catch (error) { next(error); } } async deleteUser(req: Request, res: Response, next: NextFunction) { try { const { id } = req.params; await this.userService.deleteUser(id); res.status(204).send(); } catch (error) { next(error); } } } ``` **Service Layer:** ```typescript // services/user.service.ts import { UserRepository } from "../repositories/user.repository"; import { CreateUserDTO, UpdateUserDTO, User } from "../types/user.types"; import { NotFoundError, ValidationError } from "../utils/errors"; import bcrypt from "bcrypt"; export class UserService { constructor(private userRepository: UserRepository) {} async createUser(userData: CreateUserDTO): Promise<User> { // Validation const existingUser = await this.userRepository.findByEmail(userData.email); if (existingUser) { throw new ValidationError("Email already exists"); } // Hash password const hashedPassword = await bcrypt.hash(userData.password, 10); // Create user const user = await this.userRepository.create({ ...userData, password: hashedPassword, }); // Remove password from response const { password, ...userWithoutPassword } = user; return userWithoutPassword as User; } async getUserById(id: string): Promise<User> { const user = await this.userRepository.findById(id); if (!user) { throw new NotFoundError("User not found"); } const { password, ...userWithoutPassword } = user; return userWithoutPassword as User; } async updateUser(id: string, updates: UpdateUserDTO): Promise<User> { const user = await this.userRepository.update(id, updates); if (!user) { throw new NotFoundError("User not found"); } const { password, ...userWithoutPassword } = user; return userWithoutPassword as User; } async deleteUser(id: string): Promise<void> { const deleted = await this.userRepository.delete(id); if (!deleted) { throw new NotFoundError("User not found"); } } } ``` **Repository Layer:** ```typescript // repositories/user.repository.ts import { Pool } from "pg"; import { CreateUserDTO, UpdateUserDTO, UserEntity } from "../types/user.types"; export class UserRepository { constructor(private db: Pool) {} async create( userData: CreateUserDTO & { password: string }, ): Promise<UserEntity> { const query = ` INSERT INTO users (name, email, password) VALUES ($1, $2, $3) RETURNING id, name, email, password, created_at, updated_at `; const { rows } = await this.db.query(query, [ userData.name, userData.email, userData.password, ]); return rows[0]; } async findById(id: string): Promise<UserEntity | null> { const query = "SELECT * FROM users WHERE id = $1"; const { rows } = await this.db.query(query, [id]); return rows[0] || null; } async findByEmail(email: string): Promise<UserEntity | null> { const query = "SELECT * FROM users WHERE email = $1"; const { rows } = await this.db.query(query, [email]); return rows[0] || null; } async update(id: string, updates: UpdateUserDTO): Promise<UserEntity | null> { const fields = Object.keys(updates); const values = Object.values(updates); const setClause = fields .map((field, idx) => `${field} = $${idx + 2}`) .join(", "); const query = ` UPDATE users SET ${setClause}, updated_at = CURRENT_TIMESTAMP WHERE id = $1 RETURNING * `; const { rows } = await this.db.query(query, [id, ...values]); return rows[0] || null; } async delete(id: string): Promise<boolean> { const query = "DELETE FROM users WHERE id = $1"; const { rowCount } = await this.db.query(query, [id]); return rowCount > 0; } } ``` ### Pattern 2: Dependency Injection **DI Container:** ```typescript // di-container.ts import { Pool } from "pg"; import { UserRepository } from "./repositories/user.repository"; import { UserService } from "./services/user.service"; import { UserController } from "./controllers/user.controller"; import { AuthService } from "./services/auth.service"; class Container { private instances = new Map<string, any>(); register<T>(key: string, factory: () => T): void { this.instances.set(key, factory); } resolve<T>(key: string): T { const factory = this.instances.get(key); if (!factory) { throw new Error(`No factory registered for ${key}`); } return factory(); } singleton<T>(key: string, factory: () => T): void { let instance: T; this.instances.set(key, () => { if (!instance) { instance = factory(); } return instance; }); } } export const container = new Container(); // Register dependencies container.singleton( "db", () => new Pool({ host: process.env.DB_HOST, port: parseInt(process.env.DB_PORT || "5432"), database: process.env.DB_NAME, user: process.env.DB_USER, password: process.env.DB_PASSWORD, max: 20, idleTimeoutMillis: 30000, connectionTimeoutMillis: 2000, }), ); container.singleton( "userRepository", () => new UserRepository(container.resolve("db")), ); container.singleton( "userService", () => new UserService(container.resolve("userRepository")), ); container.register( "userController", () => new UserController(container.resolve("userService")), ); container.singleton( "authService", () => new AuthService(container.resolve("userRepository")), ); ``` ## Middleware Patterns ### Authentication Middleware ```typescript // middleware/auth.middleware.ts import { Request, Response, NextFunction } from "express"; import jwt from "jsonwebtoken"; import { UnauthorizedError } from "../utils/errors"; interface JWTPayload { userId: string; email: string; } declare global { namespace Express { interface Request { user?: JWTPayload; } } } export const authenticate = async ( req: Request, res: Response, next: NextFunction, ) => { try { const token = req.headers.authorization?.replace("Bearer ", ""); if (!token) { throw new UnauthorizedError("No token provided"); } const payload = jwt.verify(token, process.env.JWT_SECRET!) as JWTPayload; req.user = payload; next(); } catch (error) { next(new UnauthorizedError("Invalid token")); } }; export const authorize = (...roles: string[]) => { return async (req: Request, res: Response, next: NextFunction) => { if (!req.user) { return next(new UnauthorizedError("Not authenticated")); } // Check if user has required role const hasRole = roles.some((role) => req.user?.roles?.includes(role)); if (!hasRole) { return next(new UnauthorizedError("Insufficient permissions")); } next(); }; }; ``` ### Validation Middleware ```typescript // middleware/validation.middleware.ts import { Request, Response, NextFunction } from "express"; import { AnyZodObject, ZodError } from "zod"; import { ValidationError } from "../utils/errors"; export const validate = (schema: AnyZodObject) => { return async (req: Request, res: Response, next: NextFunction) => { try { await schema.parseAsync({ body: req.body, query: req.query, params: req.params, }); next(); } catch (error) { if (error instanceof ZodError) { const errors = error.errors.map((err) => ({ field: err.path.join("."), message: err.message, })); next(new ValidationError("Validation failed", errors)); } else { next(error); } } }; }; // Usage with Zod import { z } from "zod"; const createUserSchema = z.object({ body: z.object({ name: z.string().min(1), email: z.string().email(), password: z.string().min(8), }), }); router.post("/users", validate(createUserSchema), userController.createUser); ``` ### Rate Limiting Middleware ```typescript // middleware/rate-limit.middleware.ts import rateLimit from "express-rate-limit"; import RedisStore from "rate-limit-redis"; import Redis from "ioredis"; const redis = new Redis({ host: process.env.REDIS_HOST, port: parseInt(process.env.REDIS_PORT || "6379"), }); export const apiLimiter = rateLimit({ store: new RedisStore({ client: redis, prefix: "rl:", }), windowMs: 15 * 60 * 1000, // 15 minutes max: 100, // Limit each IP to 100 requests per windowMs message: "Too many requests from this IP, please try again later", standardHeaders: true, legacyHeaders: false, }); export const authLimiter = rateLimit({ store: new RedisStore({ client: redis, prefix: "rl:auth:", }), windowMs: 15 * 60 * 1000, max: 5, // Stricter limit for auth endpoints skipSuccessfulRequests: true, }); ``` ### Request Logging Middleware ```typescript // middleware/logger.middleware.ts import { Request, Response, NextFunction } from "express"; import pino from "pino"; const logger = pino({ level: process.env.LOG_LEVEL || "info", transport: { target: "pino-pretty", options: { colorize: true }, }, }); export const requestLogger = ( req: Request, res: Response, next: NextFunction, ) => { const start = Date.now(); // Log response when finished res.on("finish", () => { const duration = Date.now() - start; logger.info({ method: req.method, url: req.url, status: res.statusCode, duration: `${duration}ms`, userAgent: req.headers["user-agent"], ip: req.ip, }); }); next(); }; export { logger }; ``` ## Error Handling ### Custom Error Classes ```typescript // utils/errors.ts export class AppError extends Error { constructor( public message: string, public statusCode: number = 500, public isOperational: boolean = true, ) { super(message); Object.setPrototypeOf(this, AppError.prototype); Error.captureStackTrace(this, this.constructor); } } export class ValidationError extends AppError { constructor( message: string, public errors?: any[], ) { super(message, 400); } } export class NotFoundError extends AppError { constructor(message: string = "Resource not found") { super(message, 404); } } export class UnauthorizedError extends AppError { constructor(message: string = "Unauthorized") { super(message, 401); } } export class ForbiddenError extends AppError { constructor(message: string = "Forbidden") { super(message, 403); } } export class ConflictError extends AppError { constructor(message: string) { super(message, 409); } } ``` ### Global Error Handler ```typescript // middleware/error-handler.ts import { Request, Response, NextFunction } from "express"; import { AppError } from "../utils/errors"; import { logger } from "./logger.middleware"; export const errorHandler = ( err: Error, req: Request, res: Response, next: NextFunction, ) => { if (err instanceof AppError) { return res.status(err.statusCode).json({ status: "error", message: err.message, ...(err instanceof ValidationError && { errors: err.errors }), }); } // Log unexpected errors logger.error({ error: err.message, stack: err.stack, url: req.url, method: req.method, }); // Don't leak error details in production const message = process.env.NODE_ENV === "production" ? "Internal server error" : err.message; res.status(500).json({ status: "error", message, }); }; // Async error wrapper export const asyncHandler = ( fn: (req: Request, res: Response, next: NextFunction) => Promise<any>, ) => { return (req: Request, res: Response, next: NextFunction) => { Promise.resolve(fn(req, res, next)).catch(next); }; }; ``` ## Database Patterns ### PostgreSQL with Connection Pool ```typescript // config/database.ts import { Pool, PoolConfig } from "pg"; const poolConfig: PoolConfig = { host: process.env.DB_HOST, port: parseInt(process.env.DB_PORT || "5432"), database: process.env.DB_NAME, user: process.env.DB_USER, password: process.env.DB_PASSWORD, max: 20, idleTimeoutMillis: 30000, connectionTimeoutMillis: 2000, }; export const pool = new Pool(poolConfig); // Test connection pool.on("connect", () => { console.log("Database connected"); }); pool.on("error", (err) => { console.error("Unexpected database error", err); process.exit(-1); }); // Graceful shutdown export const closeDatabase = async () => { await pool.end(); console.log("Database connection closed"); }; ``` ### MongoDB with Mongoose ```typescript // config/mongoose.ts import mongoose from "mongoose"; const connectDB = async () => { try { await mongoose.connect(process.env.MONGODB_URI!, { maxPoolSize: 10, serverSelectionTimeoutMS: 5000, socketTimeoutMS: 45000, }); console.log("MongoDB connected"); } catch (error) { console.error("MongoDB connection error:", error); process.exit(1); } }; mongoose.connection.on("disconnected", () => { console.log("MongoDB disconnected"); }); mongoose.connection.on("error", (err) => { console.error("MongoDB error:", err); }); export { connectDB }; // Model example import { Schema, model, Document } from "mongoose"; interface IUser extends Document { name: string; email: string; password: string; createdAt: Date; updatedAt: Date; } const userSchema = new Schema<IUser>( { name: { type: String, required: true }, email: { type: String, required: true, unique: true }, password: { type: String, required: true }, }, { timestamps: true, }, ); // Indexes userSchema.index({ email: 1 }); export const User = model<IUser>("User", userSchema); ``` ### Transaction Pattern ```typescript // services/order.service.ts import { Pool } from "pg"; export class OrderService { constructor(private db: Pool) {} async createOrder(userId: string, items: any[]) { const client = await this.db.connect(); try { await client.query("BEGIN"); // Create order const orderResult = await client.query( "INSERT INTO orders (user_id, total) VALUES ($1, $2) RETURNING id", [userId, calculateTotal(items)], ); const orderId = orderResult.rows[0].id; // Create order items for (const item of items) { await client.query( "INSERT INTO order_items (order_id, product_id, quantity, price) VALUES ($1, $2, $3, $4)", [orderId, item.productId, item.quantity, item.price], ); // Update inventory await client.query( "UPDATE products SET stock = stock - $1 WHERE id = $2", [item.quantity, item.productId], ); } await client.query("COMMIT"); return orderId; } catch (error) { await client.query("ROLLBACK"); throw error; } finally { client.release(); } } } ``` ## Authentication & Authorization ### JWT Authentication ```typescript // services/auth.service.ts import jwt from
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

typescript-advanced-types

Master TypeScript's advanced type system including generics,

coding
⭐1
# TypeScript Advanced Types Comprehensive guidance for mastering TypeScript's advanced type system including generics, conditional types, mapped types, template literal types, and utility types for building robust, type-safe applications. ## When to Use This Skill - Building type-safe libraries or frameworks - Creating reusable generic components - Implementing complex type inference logic - Designing type-safe API clients - Building form validation systems - Creating strongly-typed configuration objects - Implementing type-safe state management - Migrating JavaScript codebases to TypeScript ## Core Concepts ### 1. Generics **Purpose:** Create reusable, type-flexible components while maintaining type safety. **Basic Generic Function:** ```typescript function identity<T>(value: T): T { return value; } const num = identity<number>(42); // Type: number const str = identity<string>("hello"); // Type: string const auto = identity(true); // Type inferred: boolean ``` **Generic Constraints:** ```typescript interface HasLength { length: number; } function logLength<T extends HasLength>(item: T): T { console.log(item.length); return item; } logLength("hello"); // OK: string has length logLength([1, 2, 3]); // OK: array has length logLength({ length: 10 }); // OK: object has length // logLength(42); // Error: number has no length ``` **Multiple Type Parameters:** ```typescript function merge<T, U>(obj1: T, obj2: U): T & U { return { ...obj1, ...obj2 }; } const merged = merge({ name: "John" }, { age: 30 }); // Type: { name: string } & { age: number } ``` ### 2. Conditional Types **Purpose:** Create types that depend on conditions, enabling sophisticated type logic. **Basic Conditional Type:** ```typescript type IsString<T> = T extends string ? true : false; type A = IsString<string>; // true type B = IsString<number>; // false ``` **Extracting Return Types:** ```typescript type ReturnType<T> = T extends (...args: any[]) => infer R ? R : never; function getUser() { return { id: 1, name: "John" }; } type User = ReturnType<typeof getUser>; // Type: { id: number; name: string; } ``` **Distributive Conditional Types:** ```typescript type ToArray<T> = T extends any ? T[] : never; type StrOrNumArray = ToArray<string | number>; // Type: string[] | number[] ``` **Nested Conditions:** ```typescript type TypeName<T> = T extends string ? "string" : T extends number ? "number" : T extends boolean ? "boolean" : T extends undefined ? "undefined" : T extends Function ? "function" : "object"; type T1 = TypeName<string>; // "string" type T2 = TypeName<() => void>; // "function" ``` ### 3. Mapped Types **Purpose:** Transform existing types by iterating over their properties. **Basic Mapped Type:** ```typescript type Readonly<T> = { readonly [P in keyof T]: T[P]; }; interface User { id: number; name: string; } type ReadonlyUser = Readonly<User>; // Type: { readonly id: number; readonly name: string; } ``` **Optional Properties:** ```typescript type Partial<T> = { [P in keyof T]?: T[P]; }; type PartialUser = Partial<User>; // Type: { id?: number; name?: string; } ``` **Key Remapping:** ```typescript type Getters<T> = { [K in keyof T as `get${Capitalize<string & K>}`]: () => T[K]; }; interface Person { name: string; age: number; } type PersonGetters = Getters<Person>; // Type: { getName: () => string; getAge: () => number; } ``` **Filtering Properties:** ```typescript type PickByType<T, U> = { [K in keyof T as T[K] extends U ? K : never]: T[K]; }; interface Mixed { id: number; name: string; age: number; active: boolean; } type OnlyNumbers = PickByType<Mixed, number>; // Type: { id: number; age: number; } ``` ### 4. Template Literal Types **Purpose:** Create string-based types with pattern matching and transformation. **Basic Template Literal:** ```typescript type EventName = "click" | "focus" | "blur"; type EventHandler = `on${Capitalize<EventName>}`; // Type: "onClick" | "onFocus" | "onBlur" ``` **String Manipulation:** ```typescript type UppercaseGreeting = Uppercase<"hello">; // "HELLO" type LowercaseGreeting = Lowercase<"HELLO">; // "hello" type CapitalizedName = Capitalize<"john">; // "John" type UncapitalizedName = Uncapitalize<"John">; // "john" ``` **Path Building:** ```typescript type Path<T> = T extends object ? { [K in keyof T]: K extends string ? `${K}` | `${K}.${Path<T[K]>}` : never; }[keyof T] : never; interface Config { server: { host: string; port: number; }; database: { url: string; }; } type ConfigPath = Path<Config>; // Type: "server" | "database" | "server.host" | "server.port" | "database.url" ``` ### 5. Utility Types **Built-in Utility Types:** ```typescript // Partial<T> - Make all properties optional type PartialUser = Partial<User>; // Required<T> - Make all properties required type RequiredUser = Required<PartialUser>; // Readonly<T> - Make all properties readonly type ReadonlyUser = Readonly<User>; // Pick<T, K> - Select specific properties type UserName = Pick<User, "name" | "email">; // Omit<T, K> - Remove specific properties type UserWithoutPassword = Omit<User, "password">; // Exclude<T, U> - Exclude types from union type T1 = Exclude<"a" | "b" | "c", "a">; // "b" | "c" // Extract<T, U> - Extract types from union type T2 = Extract<"a" | "b" | "c", "a" | "b">; // "a" | "b" // NonNullable<T> - Exclude null and undefined type T3 = NonNullable<string | null | undefined>; // string // Record<K, T> - Create object type with keys K and values T type PageInfo = Record<"home" | "about", { title: string }>; ``` ## Advanced Patterns ### Pattern 1: Type-Safe Event Emitter ```typescript type EventMap = { "user:created": { id: string; name: string }; "user:updated": { id: string }; "user:deleted": { id: string }; }; class TypedEventEmitter<T extends Record<string, any>> { private listeners: { [K in keyof T]?: Array<(data: T[K]) => void>; } = {}; on<K extends keyof T>(event: K, callback: (data: T[K]) => void): void { if (!this.listeners[event]) { this.listeners[event] = []; } this.listeners[event]!.push(callback); } emit<K extends keyof T>(event: K, data: T[K]): void { const callbacks = this.listeners[event]; if (callbacks) { callbacks.forEach((callback) => callback(data)); } } } const emitter = new TypedEventEmitter<EventMap>(); emitter.on("user:created", (data) => { console.log(data.id, data.name); // Type-safe! }); emitter.emit("user:created", { id: "1", name: "John" }); // emitter.emit("user:created", { id: "1" }); // Error: missing 'name' ``` ### Pattern 2: Type-Safe API Client ```typescript type HTTPMethod = "GET" | "POST" | "PUT" | "DELETE"; type EndpointConfig = { "/users": { GET: { response: User[] }; POST: { body: { name: string; email: string }; response: User }; }; "/users/:id": { GET: { params: { id: string }; response: User }; PUT: { params: { id: string }; body: Partial<User>; response: User }; DELETE: { params: { id: string }; response: void }; }; }; type ExtractParams<T> = T extends { params: infer P } ? P : never; type ExtractBody<T> = T extends { body: infer B } ? B : never; type ExtractResponse<T> = T extends { response: infer R } ? R : never; class APIClient<Config extends Record<string, Record<HTTPMethod, any>>> { async request<Path extends keyof Config, Method extends keyof Config[Path]>( path: Path, method: Method, ...[options]: ExtractParams<Config[Path][Method]> extends never ? ExtractBody<Config[Path][Method]> extends never ? [] : [{ body: ExtractBody<Config[Path][Method]> }] : [ { params: ExtractParams<Config[Path][Method]>; body?: ExtractBody<Config[Path][Method]>; }, ] ): Promise<ExtractResponse<Config[Path][Method]>> { // Implementation here return {} as any; } } const api = new APIClient<EndpointConfig>(); // Type-safe API calls const users = await api.request("/users", "GET"); // Type: User[] const newUser = await api.request("/users", "POST", { body: { name: "John", email: "john@example.com" }, }); // Type: User const user = await api.request("/users/:id", "GET", { params: { id: "123" }, }); // Type: User ``` ### Pattern 3: Builder Pattern with Type Safety ```typescript type BuilderState<T> = { [K in keyof T]: T[K] | undefined; }; type RequiredKeys<T> = { [K in keyof T]-?: {} extends Pick<T, K> ? never : K; }[keyof T]; type OptionalKeys<T> = { [K in keyof T]-?: {} extends Pick<T, K> ? K : never; }[keyof T]; type IsComplete<T, S> = RequiredKeys<T> extends keyof S ? S[RequiredKeys<T>] extends undefined ? false : true : false; class Builder<T, S extends BuilderState<T> = {}> { private state: S = {} as S; set<K extends keyof T>(key: K, value: T[K]): Builder<T, S & Record<K, T[K]>> { this.state[key] = value; return this as any; } build(this: IsComplete<T, S> extends true ? this : never): T { return this.state as T; } } interface User { id: string; name: string; email: string; age?: number; } const builder = new Builder<User>(); const user = builder .set("id", "1") .set("name", "John") .set("email", "john@example.com") .build(); // OK: all required fields set // const incomplete = builder // .set("id", "1") // .build(); // Error: missing required fields ``` ### Pattern 4: Deep Readonly/Partial ```typescript type DeepReadonly<T> = { readonly [P in keyof T]: T[P] extends object ? T[P] extends Function ? T[P] : DeepReadonly<T[P]> : T[P]; }; type DeepPartial<T> = { [P in keyof T]?: T[P] extends object ? T[P] extends Array<infer U> ? Array<DeepPartial<U>> : DeepPartial<T[P]> : T[P]; }; interface Config { server: { host: string; port: number; ssl: { enabled: boolean; cert: string; }; }; database: { url: string; pool: { min: number; max: number; }; }; } type ReadonlyConfig = DeepReadonly<Config>; // All nested properties are readonly type PartialConfig = DeepPartial<Config>; // All nested properties are optional ``` ### Pattern 5: Type-Safe Form Validation ```typescript type ValidationRule<T> = { validate: (value: T) => boolean; message: string; }; type FieldValidation<T> = { [K in keyof T]?: ValidationRule<T[K]>[]; }; type ValidationErrors<T> = { [K in keyof T]?: string[]; }; class FormValidator<T extends Record<string, any>> { constructor(private rules: FieldValidation<T>) {} validate(data: T): ValidationErrors<T> | null { const errors: ValidationErrors<T> = {}; let hasErrors = false; for (const key in this.rules) { const fieldRules = this.rules[key]; const value = data[key]; if (fieldRules) { const fieldErrors: string[] = []; for (const rule of fieldRules) { if (!rule.validate(value)) { fieldErrors.push(rule.message); } } if (fieldErrors.length > 0) { errors[key] = fieldErrors; hasErrors = true; } } } return hasErrors ? errors : null; } } interface LoginForm { email: string; password: string; } const validator = new FormValidator<LoginForm>({ email: [ { validate: (v) => v.includes("@"), message: "Email must contain @", }, { validate: (v) => v.length > 0, message: "Email is required", }, ], password: [ { validate: (v) => v.length >= 8, message: "Password must be at least 8 characters", }, ], }); const errors = validator.validate({ email: "invalid", password: "short", }); // Type: { email?: string[]; password?: string[]; } | null ``` ### Pattern 6: Discriminated Unions ```typescript type Success<T> = { status: "success"; data: T; }; type Error = { status: "error"; error: string; }; type Loading = { status: "loading"; }; type AsyncState<T> = Success<T> | Error | Loading; function handleState<T>(state: AsyncState<T>): void { switch (state.status) { case "success": console.log(state.data); // Type: T break; case "error": console.log(state.error); // Type: string break; case "loading": console.log("Loading..."); break; } } // Type-safe state machine type State = | { type: "idle" } | { type: "fetching"; requestId: string } | { type: "success"; data: any } | { type: "error"; error: Error }; type Event = | { type: "FETCH"; requestId: string } | { type: "SUCCESS"; data: any } | { type: "ERROR"; error: Error } | { type: "RESET" }; function reducer(state: State, event: Event): State { switch (state.type) { case "idle": return event.type === "FETCH" ? { type: "fetching", requestId: event.requestId } : state; case "fetching": if (event.type === "SUCCESS") { return { type: "success", data: event.data }; } if (event.type === "ERROR") { return { type: "error", error: event.error }; } return state; case "success": case "error": return event.type === "RESET" ? { type: "idle" } : state; } } ``` ## Type Inference Techniques ### 1. Infer Keyword ```typescript // Extract array element type type ElementType<T> = T extends (infer U)[] ? U : never; type NumArray = number[]; type Num = ElementType<NumArray>; // number // Extract promise type type PromiseType<T> = T extends Promise<infer U> ? U : never; type AsyncNum = PromiseType<Promise<number>>; // number // Extract function parameters type Parameters<T> = T extends (...args: infer P) => any ? P : never; function foo(a: string, b: number) {} type FooParams = Parameters<typeof foo>; // [string, number] ``` ### 2. Type Guards ```typescript function isString(value: unknown): value is string { return typeof value === "string"; } function isArrayOf<T>( value: unknown, guard: (item: unknown) => item is T, ): value is T[] { return Array.isArray(value) && value.every(guard); } const data: unknown = ["a", "b", "c"]; if (isArrayOf(data, isString)) { data.forEach((s) => s.toUpperCase()); // Type: string[] } ``` ### 3. Assertion Functions ```typescript function assertIsString(value: unknown): asserts value is string { if (typeof value !== "string") { throw new Error("Not a string"); } } function processValue(value: unknown) { assertIsString(value); // value is now typed as string console.log(value.toUpperCase()); } ``` ## Best Practices 1. **Use `unknown` over `any`**: Enforce type checking 2. **Prefer `interface` for object shapes**: Better error messages 3. **Use `type` for unions and complex types**: More flexible 4. **Leverage type inference**: Let TypeScript infer when possible 5. **Create helper types**: Build reusable type utilities 6. **Use const assertions**: Preserve literal types 7. **Avoid type assertions**: Use type guards instead 8. **Document complex types**: Add JSDoc comments 9. **Use strict mode**: Enable all strict compiler options 10. **Test your types**: Use type tests to verify type behavior ## Type Testing ```typescript // Type assertion tests type AssertEqual<T, U> = [T] extends [U] ? [U] extends [T] ? true : false : false; type Test1 = AssertEqual<string, string>; // true type Test2 = AssertEqual<string, number>; // false type Test3 = AssertEqual<string | number, string>; // false // Expect error helper type ExpectError<T extends never> = T; // Example usage type ShouldError = ExpectError<AssertEqual<string, number>>; ``` ## Common Pitfalls 1. **Over-using `any`**: Defeats the purpose of TypeScript 2. **Ignoring strict null checks**: Can lead to runtime errors 3. **Too complex types**: Can slow down compilation 4. **Not using discriminated unions**: Misses type narrowing opportunities 5. **Forgetting readonly modifiers**: Allows unintended mutations 6. **Circular type references**: Can cause compiler errors 7. **Not handling edge cases**: Like empty arrays or null values ## Performance Considerations - Avoid deeply nested conditional types - Use simple types when possible - Cache complex type computations - Limit recursion depth in recursive types - Use build tools to skip type checking in production ## Resources - **TypeScript Handbook**: https://www.typescriptlang.org/docs/handbook/ - **Type Challenges**: https://github.com/type-challenges/type-challenges - **TypeScript Deep Dive**: https://basarat.gitbook.io/typescript/ - **Effective TypeScript**: Book by Dan Vanderkam
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

prompt-engineering-patterns

Master advanced prompt engineering techniques to maximize LLM

coding
⭐1
# Prompt Engineering Patterns Master advanced prompt engineering techniques to maximize LLM performance, reliability, and controllability. ## When to Use This Skill - Designing complex prompts for production LLM applications - Optimizing prompt performance and consistency - Implementing structured reasoning patterns (chain-of-thought, tree-of-thought) - Building few-shot learning systems with dynamic example selection - Creating reusable prompt templates with variable interpolation - Debugging and refining prompts that produce inconsistent outputs - Implementing system prompts for specialized AI assistants - Using structured outputs (JSON mode) for reliable parsing ## Core Capabilities ### 1. Few-Shot Learning - Example selection strategies (semantic similarity, diversity sampling) - Balancing example count with context window constraints - Constructing effective demonstrations with input-output pairs - Dynamic example retrieval from knowledge bases - Handling edge cases through strategic example selection ### 2. Chain-of-Thought Prompting - Step-by-step reasoning elicitation - Zero-shot CoT with "Let's think step by step" - Few-shot CoT with reasoning traces - Self-consistency techniques (sampling multiple reasoning paths) - Verification and validation steps ### 3. Structured Outputs - JSON mode for reliable parsing - Pydantic schema enforcement - Type-safe response handling - Error handling for malformed outputs ### 4. Prompt Optimization - Iterative refinement workflows - A/B testing prompt variations - Measuring prompt performance metrics (accuracy, consistency, latency) - Reducing token usage while maintaining quality - Handling edge cases and failure modes ### 5. Template Systems - Variable interpolation and formatting - Conditional prompt sections - Multi-turn conversation templates - Role-based prompt composition - Modular prompt components ### 6. System Prompt Design - Setting model behavior and constraints - Defining output formats and structure - Establishing role and expertise - Safety guidelines and content policies - Context setting and background information ## Quick Start ```python from langchain_anthropic import ChatAnthropic from langchain_core.prompts import ChatPromptTemplate from pydantic import BaseModel, Field # Define structured output schema class SQLQuery(BaseModel): query: str = Field(description="The SQL query") explanation: str = Field(description="Brief explanation of what the query does") tables_used: list[str] = Field(description="List of tables referenced") # Initialize model with structured output llm = ChatAnthropic(model="claude-sonnet-4-6") structured_llm = llm.with_structured_output(SQLQuery) # Create prompt template prompt = ChatPromptTemplate.from_messages([ ("system", """You are an expert SQL developer. Generate efficient, secure SQL queries. Always use parameterized queries to prevent SQL injection. Explain your reasoning briefly."""), ("user", "Convert this to SQL: {query}") ]) # Create chain chain = prompt | structured_llm # Use result = await chain.ainvoke({ "query": "Find all users who registered in the last 30 days" }) print(result.query) print(result.explanation) ``` ## Key Patterns ### Pattern 1: Structured Output with Pydantic ```python from anthropic import Anthropic from pydantic import BaseModel, Field from typing import Literal import json class SentimentAnalysis(BaseModel): sentiment: Literal["positive", "negative", "neutral"] confidence: float = Field(ge=0, le=1) key_phrases: list[str] reasoning: str async def analyze_sentiment(text: str) -> SentimentAnalysis: """Analyze sentiment with structured output.""" client = Anthropic() message = client.messages.create( model="claude-sonnet-4-6", max_tokens=500, messages=[{ "role": "user", "content": f"""Analyze the sentiment of this text. Text: {text} Respond with JSON matching this schema: {{ "sentiment": "positive" | "negative" | "neutral", "confidence": 0.0-1.0, "key_phrases": ["phrase1", "phrase2"], "reasoning": "brief explanation" }}""" }] ) return SentimentAnalysis(**json.loads(message.content[0].text)) ``` ### Pattern 2: Chain-of-Thought with Self-Verification ```python from langchain_core.prompts import ChatPromptTemplate cot_prompt = ChatPromptTemplate.from_template(""" Solve this problem step by step. Problem: {problem} Instructions: 1. Break down the problem into clear steps 2. Work through each step showing your reasoning 3. State your final answer 4. Verify your answer by checking it against the original problem Format your response as: ## Steps [Your step-by-step reasoning] ## Answer [Your final answer] ## Verification [Check that your answer is correct] """) ``` ### Pattern 3: Few-Shot with Dynamic Example Selection ```python from langchain_voyageai import VoyageAIEmbeddings from langchain_core.example_selectors import SemanticSimilarityExampleSelector from langchain_chroma import Chroma # Create example selector with semantic similarity example_selector = SemanticSimilarityExampleSelector.from_examples( examples=[ {"input": "How do I reset my password?", "output": "Go to Settings > Security > Reset Password"}, {"input": "Where can I see my order history?", "output": "Navigate to Account > Orders"}, {"input": "How do I contact support?", "output": "Click Help > Contact Us or email support@example.com"}, ], embeddings=VoyageAIEmbeddings(model="voyage-3-large"), vectorstore_cls=Chroma, k=2 # Select 2 most similar examples ) async def get_few_shot_prompt(query: str) -> str: """Build prompt with dynamically selected examples.""" examples = await example_selector.aselect_examples({"input": query}) examples_text = "\n".join( f"User: {ex['input']}\nAssistant: {ex['output']}" for ex in examples ) return f"""You are a helpful customer support assistant. Here are some example interactions: {examples_text} Now respond to this query: User: {query} Assistant:""" ``` ### Pattern 4: Progressive Disclosure Start with simple prompts, add complexity only when needed: ```python PROMPT_LEVELS = { # Level 1: Direct instruction "simple": "Summarize this article: {text}", # Level 2: Add constraints "constrained": """Summarize this article in 3 bullet points, focusing on: - Key findings - Main conclusions - Practical implications Article: {text}""", # Level 3: Add reasoning "reasoning": """Read this article carefully. 1. First, identify the main topic and thesis 2. Then, extract the key supporting points 3. Finally, summarize in 3 bullet points Article: {text} Summary:""", # Level 4: Add examples "few_shot": """Read articles and provide concise summaries. Example: Article: "New research shows that regular exercise can reduce anxiety by up to 40%..." Summary: • Regular exercise reduces anxiety by up to 40% • 30 minutes of moderate activity 3x/week is sufficient • Benefits appear within 2 weeks of starting Now summarize this article: Article: {text} Summary:""" } ``` ### Pattern 5: Error Recovery and Fallback ```python from pydantic import BaseModel, ValidationError import json class ResponseWithConfidence(BaseModel): answer: str confidence: float sources: list[str] alternative_interpretations: list[str] = [] ERROR_RECOVERY_PROMPT = """ Answer the question based on the context provided. Context: {context} Question: {question} Instructions: 1. If you can answer confidently (>0.8), provide a direct answer 2. If you're somewhat confident (0.5-0.8), provide your best answer with caveats 3. If you're uncertain (<0.5), explain what information is missing 4. Always provide alternative interpretations if the question is ambiguous Respond in JSON: {{ "answer": "your answer or 'I cannot determine this from the context'", "confidence": 0.0-1.0, "sources": ["relevant context excerpts"], "alternative_interpretations": ["if question is ambiguous"] }} """ async def answer_with_fallback( context: str, question: str, llm ) -> ResponseWithConfidence: """Answer with error recovery and fallback.""" prompt = ERROR_RECOVERY_PROMPT.format(context=context, question=question) try: response = await llm.ainvoke(prompt) return ResponseWithConfidence(**json.loads(response.content)) except (json.JSONDecodeError, ValidationError) as e: # Fallback: try to extract answer without structure simple_prompt = f"Based on: {context}\n\nAnswer: {question}" simple_response = await llm.ainvoke(simple_prompt) return ResponseWithConfidence( answer=simple_response.content, confidence=0.5, sources=["fallback extraction"], alternative_interpretations=[] ) ``` ### Pattern 6: Role-Based System Prompts ```python SYSTEM_PROMPTS = { "analyst": """You are a senior data analyst with expertise in SQL, Python, and business intelligence. Your responsibilities: - Write efficient, well-documented queries - Explain your analysis methodology - Highlight key insights and recommendations - Flag any data quality concerns Communication style: - Be precise and technical when discussing methodology - Translate technical findings into business impact - Use clear visualizations when helpful""", "assistant": """You are a helpful AI assistant focused on accuracy and clarity. Core principles: - Always cite sources when making factual claims - Acknowledge uncertainty rather than guessing - Ask clarifying questions when the request is ambiguous - Provide step-by-step explanations for complex topics Constraints: - Do not provide medical, legal, or financial advice - Redirect harmful requests appropriately - Protect user privacy""", "code_reviewer": """You are a senior software engineer conducting code reviews. Review criteria: - Correctness: Does the code work as intended? - Security: Are there any vulnerabilities? - Performance: Are there efficiency concerns? - Maintainability: Is the code readable and well-structured? - Best practices: Does it follow language idioms? Output format: 1. Summary assessment (approve/request changes) 2. Critical issues (must fix) 3. Suggestions (nice to have) 4. Positive feedback (what's done well)""" } ``` ## Integration Patterns ### With RAG Systems ```python RAG_PROMPT = """You are a knowledgeable assistant that answers questions based on provided context. Context (retrieved from knowledge base): {context} Instructions: 1. Answer ONLY based on the provided context 2. If the context doesn't contain the answer, say "I don't have information about that in my knowledge base" 3. Cite specific passages using [1], [2] notation 4. If the question is ambiguous, ask for clarification Question: {question} Answer:""" ``` ### With Validation and Verification ```python VALIDATED_PROMPT = """Complete the following task: Task: {task} After generating your response, verify it meets ALL these criteria: āœ“ Directly addresses the original request āœ“ Contains no factual errors āœ“ Is appropriately detailed (not too brief, not too verbose) āœ“ Uses proper formatting āœ“ Is safe and appropriate If verification fails on any criterion, revise before responding. Response:""" ``` ## Performance Optimization ### Token Efficiency ```python # Before: Verbose prompt (150+ tokens) verbose_prompt = """ I would like you to please take the following text and provide me with a comprehensive summary of the main points. The summary should capture the key ideas and important details while being concise and easy to understand. """ # After: Concise prompt (30 tokens) concise_prompt = """Summarize the key points concisely: {text} Summary:""" ``` ### Caching Common Prefixes ```python from anthropic import Anthropic client = Anthropic() # Use prompt caching for repeated system prompts response = client.messages.create( model="claude-sonnet-4-6", max_tokens=1000, system=[ { "type": "text", "text": LONG_SYSTEM_PROMPT, "cache_control": {"type": "ephemeral"} } ], messages=[{"role": "user", "content": user_query}] ) ``` ## Best Practices 1. **Be Specific**: Vague prompts produce inconsistent results 2. **Show, Don't Tell**: Examples are more effective than descriptions 3. **Use Structured Outputs**: Enforce schemas with Pydantic for reliability 4. **Test Extensively**: Evaluate on diverse, representative inputs 5. **Iterate Rapidly**: Small changes can have large impacts 6. **Monitor Performance**: Track metrics in production 7. **Version Control**: Treat prompts as code with proper versioning 8. **Document Intent**: Explain why prompts are structured as they are ## Common Pitfalls - **Over-engineering**: Starting with complex prompts before trying simple ones - **Example pollution**: Using examples that don't match the target task - **Context overflow**: Exceeding token limits with excessive examples - **Ambiguous instructions**: Leaving room for multiple interpretations - **Ignoring edge cases**: Not testing on unusual or boundary inputs - **No error handling**: Assuming outputs will always be well-formed - **Hardcoded values**: Not parameterizing prompts for reuse ## Success Metrics Track these KPIs for your prompts: - **Accuracy**: Correctness of outputs - **Consistency**: Reproducibility across similar inputs - **Latency**: Response time (P50, P95, P99) - **Token Usage**: Average tokens per request - **Success Rate**: Percentage of valid, parseable outputs - **User Satisfaction**: Ratings and feedback ## Resources - [Anthropic Prompt Engineering Guide](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering) - [Claude Prompt Caching](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching) - [OpenAI Prompt Engineering](https://platform.openai.com/docs/guides/prompt-engineering) - [LangChain Prompts](https://python.langchain.com/docs/concepts/prompts/)
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

similarity-search-patterns

Implement efficient similarity search with vector databases. Use

coding
⭐1
# Similarity Search Patterns Patterns for implementing efficient similarity search in production systems. ## When to Use This Skill - Building semantic search systems - Implementing RAG retrieval - Creating recommendation engines - Optimizing search latency - Scaling to millions of vectors - Combining semantic and keyword search ## Core Concepts ### 1. Distance Metrics | Metric | Formula | Best For | | ------------------ | ------------------ | --------------------- | --- | -------------- | | **Cosine** | 1 - (AĀ·B)/(‖A‖‖B‖) | Normalized embeddings | | **Euclidean (L2)** | √Σ(a-b)² | Raw embeddings | | **Dot Product** | AĀ·B | Magnitude matters | | **Manhattan (L1)** | Ī£ | a-b | | Sparse vectors | ### 2. Index Types ``` ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ Index Types │ ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ │ Flat │ HNSW │ IVF+PQ │ │ (Exact) │ (Graph-based) │ (Quantized) │ ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ │ O(n) search │ O(log n) │ O(√n) │ │ 100% recall │ ~95-99% │ ~90-95% │ │ Small data │ Medium-Large │ Very Large │ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ``` ## Templates ### Template 1: Pinecone Implementation ```python from pinecone import Pinecone, ServerlessSpec from typing import List, Dict, Optional import hashlib class PineconeVectorStore: def __init__( self, api_key: str, index_name: str, dimension: int = 1536, metric: str = "cosine" ): self.pc = Pinecone(api_key=api_key) # Create index if not exists if index_name not in self.pc.list_indexes().names(): self.pc.create_index( name=index_name, dimension=dimension, metric=metric, spec=ServerlessSpec(cloud="aws", region="us-east-1") ) self.index = self.pc.Index(index_name) def upsert( self, vectors: List[Dict], namespace: str = "" ) -> int: """ Upsert vectors. vectors: [{"id": str, "values": List[float], "metadata": dict}] """ # Batch upsert batch_size = 100 total = 0 for i in range(0, len(vectors), batch_size): batch = vectors[i:i + batch_size] self.index.upsert(vectors=batch, namespace=namespace) total += len(batch) return total def search( self, query_vector: List[float], top_k: int = 10, namespace: str = "", filter: Optional[Dict] = None, include_metadata: bool = True ) -> List[Dict]: """Search for similar vectors.""" results = self.index.query( vector=query_vector, top_k=top_k, namespace=namespace, filter=filter, include_metadata=include_metadata ) return [ { "id": match.id, "score": match.score, "metadata": match.metadata } for match in results.matches ] def search_with_rerank( self, query: str, query_vector: List[float], top_k: int = 10, rerank_top_n: int = 50, namespace: str = "" ) -> List[Dict]: """Search and rerank results.""" # Over-fetch for reranking initial_results = self.search( query_vector, top_k=rerank_top_n, namespace=namespace ) # Rerank with cross-encoder or LLM reranked = self._rerank(query, initial_results) return reranked[:top_k] def _rerank(self, query: str, results: List[Dict]) -> List[Dict]: """Rerank results using cross-encoder.""" from sentence_transformers import CrossEncoder model = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') pairs = [(query, r["metadata"]["text"]) for r in results] scores = model.predict(pairs) for result, score in zip(results, scores): result["rerank_score"] = float(score) return sorted(results, key=lambda x: x["rerank_score"], reverse=True) def delete(self, ids: List[str], namespace: str = ""): """Delete vectors by ID.""" self.index.delete(ids=ids, namespace=namespace) def delete_by_filter(self, filter: Dict, namespace: str = ""): """Delete vectors matching filter.""" self.index.delete(filter=filter, namespace=namespace) ``` ### Template 2: Qdrant Implementation ```python from qdrant_client import QdrantClient from qdrant_client.http import models from typing import List, Dict, Optional class QdrantVectorStore: def __init__( self, url: str = "localhost", port: int = 6333, collection_name: str = "documents", vector_size: int = 1536 ): self.client = QdrantClient(url=url, port=port) self.collection_name = collection_name # Create collection if not exists collections = self.client.get_collections().collections if collection_name not in [c.name for c in collections]: self.client.create_collection( collection_name=collection_name, vectors_config=models.VectorParams( size=vector_size, distance=models.Distance.COSINE ), # Optional: enable quantization for memory efficiency quantization_config=models.ScalarQuantization( scalar=models.ScalarQuantizationConfig( type=models.ScalarType.INT8, quantile=0.99, always_ram=True ) ) ) def upsert(self, points: List[Dict]) -> int: """ Upsert points. points: [{"id": str/int, "vector": List[float], "payload": dict}] """ qdrant_points = [ models.PointStruct( id=p["id"], vector=p["vector"], payload=p.get("payload", {}) ) for p in points ] self.client.upsert( collection_name=self.collection_name, points=qdrant_points ) return len(points) def search( self, query_vector: List[float], limit: int = 10, filter: Optional[models.Filter] = None, score_threshold: Optional[float] = None ) -> List[Dict]: """Search for similar vectors.""" results = self.client.search( collection_name=self.collection_name, query_vector=query_vector, limit=limit, query_filter=filter, score_threshold=score_threshold ) return [ { "id": r.id, "score": r.score, "payload": r.payload } for r in results ] def search_with_filter( self, query_vector: List[float], must_conditions: List[Dict] = None, should_conditions: List[Dict] = None, must_not_conditions: List[Dict] = None, limit: int = 10 ) -> List[Dict]: """Search with complex filters.""" conditions = [] if must_conditions: conditions.extend([ models.FieldCondition( key=c["key"], match=models.MatchValue(value=c["value"]) ) for c in must_conditions ]) filter = models.Filter(must=conditions) if conditions else None return self.search(query_vector, limit=limit, filter=filter) def search_with_sparse( self, dense_vector: List[float], sparse_vector: Dict[int, float], limit: int = 10, dense_weight: float = 0.7 ) -> List[Dict]: """Hybrid search with dense and sparse vectors.""" # Requires collection with named vectors results = self.client.search( collection_name=self.collection_name, query_vector=models.NamedVector( name="dense", vector=dense_vector ), limit=limit ) return [{"id": r.id, "score": r.score, "payload": r.payload} for r in results] ``` ### Template 3: pgvector with PostgreSQL ```python import asyncpg from typing import List, Dict, Optional import numpy as np class PgVectorStore: def __init__(self, connection_string: str): self.connection_string = connection_string async def init(self): """Initialize connection pool and extension.""" self.pool = await asyncpg.create_pool(self.connection_string) async with self.pool.acquire() as conn: # Enable extension await conn.execute("CREATE EXTENSION IF NOT EXISTS vector") # Create table await conn.execute(""" CREATE TABLE IF NOT EXISTS documents ( id TEXT PRIMARY KEY, content TEXT, metadata JSONB, embedding vector(1536) ) """) # Create index (HNSW for better performance) await conn.execute(""" CREATE INDEX IF NOT EXISTS documents_embedding_idx ON documents USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64) """) async def upsert(self, documents: List[Dict]): """Upsert documents with embeddings.""" async with self.pool.acquire() as conn: await conn.executemany( """ INSERT INTO documents (id, content, metadata, embedding) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content, metadata = EXCLUDED.metadata, embedding = EXCLUDED.embedding """, [ ( doc["id"], doc["content"], doc.get("metadata", {}), np.array(doc["embedding"]).tolist() ) for doc in documents ] ) async def search( self, query_embedding: List[float], limit: int = 10, filter_metadata: Optional[Dict] = None ) -> List[Dict]: """Search for similar documents.""" query = """ SELECT id, content, metadata, 1 - (embedding <=> $1::vector) as similarity FROM documents """ params = [query_embedding] if filter_metadata: conditions = [] for key, value in filter_metadata.items(): params.append(value) conditions.append(f"metadata->>'{key}' = ${len(params)}") query += " WHERE " + " AND ".join(conditions) query += f" ORDER BY embedding <=> $1::vector LIMIT ${len(params) + 1}" params.append(limit) async with self.pool.acquire() as conn: rows = await conn.fetch(query, *params) return [ { "id": row["id"], "content": row["content"], "metadata": row["metadata"], "score": row["similarity"] } for row in rows ] async def hybrid_search( self, query_embedding: List[float], query_text: str, limit: int = 10, vector_weight: float = 0.5 ) -> List[Dict]: """Hybrid search combining vector and full-text.""" async with self.pool.acquire() as conn: rows = await conn.fetch( """ WITH vector_results AS ( SELECT id, content, metadata, 1 - (embedding <=> $1::vector) as vector_score FROM documents ORDER BY embedding <=> $1::vector LIMIT $3 * 2 ), text_results AS ( SELECT id, content, metadata, ts_rank(to_tsvector('english', content), plainto_tsquery('english', $2)) as text_score FROM documents WHERE to_tsvector('english', content) @@ plainto_tsquery('english', $2) LIMIT $3 * 2 ) SELECT COALESCE(v.id, t.id) as id, COALESCE(v.content, t.content) as content, COALESCE(v.metadata, t.metadata) as metadata, COALESCE(v.vector_score, 0) * $4 + COALESCE(t.text_score, 0) * (1 - $4) as combined_score FROM vector_results v FULL OUTER JOIN text_results t ON v.id = t.id ORDER BY combined_score DESC LIMIT $3 """, query_embedding, query_text, limit, vector_weight ) return [dict(row) for row in rows] ``` ### Template 4: Weaviate Implementation ```python import weaviate from weaviate.util import generate_uuid5 from typing import List, Dict, Optional class WeaviateVectorStore: def __init__( self, url: str = "http://localhost:8080", class_name: str = "Document" ): self.client = weaviate.Client(url=url) self.class_name = class_name self._ensure_schema() def _ensure_schema(self): """Create schema if not exists.""" schema = { "class": self.class_name, "vectorizer": "none", # We provide vectors "properties": [ {"name": "content", "dataType": ["text"]}, {"name": "source", "dataType": ["string"]}, {"name": "chunk_id", "dataType": ["int"]} ] } if not self.client.schema.exists(self.class_name): self.client.schema.create_class(schema) def upsert(self, documents: List[Dict]): """Batch upsert documents.""" with self.client.batch as batch: batch.batch_size = 100 for doc in documents: batch.add_data_object( data_object={ "content": doc["content"], "source": doc.get("source", ""), "chunk_id": doc.get("chunk_id", 0) }, class_name=self.class_name, uuid=generate_uuid5(doc["id"]), vector=doc["embedding"] ) def search( self, query_vector: List[float], limit: int = 10, where_filter: Optional[Dict] = None ) -> List[Dict]: """Vector search.""" query = ( self.client.query .get(self.class_name, ["content", "source", "chunk_id"]) .with_near_vector({"vector": query_vector}) .with_limit(limit) .with_additional(["distance", "id"]) ) if where_filter: query = query.with_where(where_filter) results = query.do() return [ { "id": item["_additional"]["id"], "content": item["content"], "source": item["source"], "score": 1 - item["_additional"]["distance"] } for item in results["data"]["Get"][self.class_name] ] def hybrid_search( self, query: str, query_vector: List[float], limit: int = 10, alpha: float = 0.5 # 0 = keyword, 1 = vector ) -> List[Dict]: """Hybrid search combining BM25 and vector.""" results = ( self.client.query .get(self.class_name, ["content", "source"]) .with_hybrid(query=query, vector=query_vector, alpha=alpha) .with_limit(limit) .with_additional(["score"]) .do() ) return [ { "content": item["content"], "source": item["source"], "score": item["_additional"]["score"] } for item in results["data"]["Get"][self.class_name] ] ``` ## Best Practices ### Do's - **Use appropriate index** - HNSW for most cases - **Tune parameters** - ef_search, nprobe for recall/speed - **Implement hybrid search** - Combine with keyword search - **Monitor recall** - Measure search quality - **Pre-filter when possible** - Reduce search space ### Don'ts - **Don't skip evaluation** - Measure before optimizing - **Don't over-index** - Start with flat, scale up - **Don't ignore latency** - P99 matters for UX - **Don't forget costs** - Vector storage adds up ## Resources - [Pinecone Docs](https://docs.pinecone.io/) - [Qdrant Docs](https://qdrant.tech/documentation/) - [pgvector](https://github.com/pgvector/pgvector) - [Weaviate Docs](https://weaviate.io/developers/weaviate)
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

vector-index-tuning

Optimize vector index performance for latency, recall, and memory.

coding
⭐1
# Vector Index Tuning Guide to optimizing vector indexes for production performance. ## When to Use This Skill - Tuning HNSW parameters - Implementing quantization - Optimizing memory usage - Reducing search latency - Balancing recall vs speed - Scaling to billions of vectors ## Core Concepts ### 1. Index Type Selection ``` Data Size Recommended Index ──────────────────────────────────────── < 10K vectors → Flat (exact search) 10K - 1M → HNSW 1M - 100M → HNSW + Quantization > 100M → IVF + PQ or DiskANN ``` ### 2. HNSW Parameters | Parameter | Default | Effect | | ------------------ | ------- | ---------------------------------------------------- | | **M** | 16 | Connections per node, ↑ = better recall, more memory | | **efConstruction** | 100 | Build quality, ↑ = better index, slower build | | **efSearch** | 50 | Search quality, ↑ = better recall, slower search | ### 3. Quantization Types ``` Full Precision (FP32): 4 bytes Ɨ dimensions Half Precision (FP16): 2 bytes Ɨ dimensions INT8 Scalar: 1 byte Ɨ dimensions Product Quantization: ~32-64 bytes total Binary: dimensions/8 bytes ``` ## Templates ### Template 1: HNSW Parameter Tuning ```python import numpy as np from typing import List, Tuple import time def benchmark_hnsw_parameters( vectors: np.ndarray, queries: np.ndarray, ground_truth: np.ndarray, m_values: List[int] = [8, 16, 32, 64], ef_construction_values: List[int] = [64, 128, 256], ef_search_values: List[int] = [32, 64, 128, 256] ) -> List[dict]: """Benchmark different HNSW configurations.""" import hnswlib results = [] dim = vectors.shape[1] n = vectors.shape[0] for m in m_values: for ef_construction in ef_construction_values: # Build index index = hnswlib.Index(space='cosine', dim=dim) index.init_index(max_elements=n, M=m, ef_construction=ef_construction) build_start = time.time() index.add_items(vectors) build_time = time.time() - build_start # Get memory usage memory_bytes = index.element_count * ( dim * 4 + # Vector storage m * 2 * 4 # Graph edges (approximate) ) for ef_search in ef_search_values: index.set_ef(ef_search) # Measure search search_start = time.time() labels, distances = index.knn_query(queries, k=10) search_time = time.time() - search_start # Calculate recall recall = calculate_recall(labels, ground_truth, k=10) results.append({ "M": m, "ef_construction": ef_construction, "ef_search": ef_search, "build_time_s": build_time, "search_time_ms": search_time * 1000 / len(queries), "recall@10": recall, "memory_mb": memory_bytes / 1024 / 1024 }) return results def calculate_recall(predictions: np.ndarray, ground_truth: np.ndarray, k: int) -> float: """Calculate recall@k.""" correct = 0 for pred, truth in zip(predictions, ground_truth): correct += len(set(pred[:k]) & set(truth[:k])) return correct / (len(predictions) * k) def recommend_hnsw_params( num_vectors: int, target_recall: float = 0.95, max_latency_ms: float = 10, available_memory_gb: float = 8 ) -> dict: """Recommend HNSW parameters based on requirements.""" # Base recommendations if num_vectors < 100_000: m = 16 ef_construction = 100 elif num_vectors < 1_000_000: m = 32 ef_construction = 200 else: m = 48 ef_construction = 256 # Adjust ef_search based on recall target if target_recall >= 0.99: ef_search = 256 elif target_recall >= 0.95: ef_search = 128 else: ef_search = 64 return { "M": m, "ef_construction": ef_construction, "ef_search": ef_search, "notes": f"Estimated for {num_vectors:,} vectors, {target_recall:.0%} recall" } ``` ### Template 2: Quantization Strategies ```python import numpy as np from typing import Optional class VectorQuantizer: """Quantization strategies for vector compression.""" @staticmethod def scalar_quantize_int8( vectors: np.ndarray, min_val: Optional[float] = None, max_val: Optional[float] = None ) -> Tuple[np.ndarray, dict]: """Scalar quantization to INT8.""" if min_val is None: min_val = vectors.min() if max_val is None: max_val = vectors.max() # Scale to 0-255 range scale = 255.0 / (max_val - min_val) quantized = np.clip( np.round((vectors - min_val) * scale), 0, 255 ).astype(np.uint8) params = {"min_val": min_val, "max_val": max_val, "scale": scale} return quantized, params @staticmethod def dequantize_int8( quantized: np.ndarray, params: dict ) -> np.ndarray: """Dequantize INT8 vectors.""" return quantized.astype(np.float32) / params["scale"] + params["min_val"] @staticmethod def product_quantize( vectors: np.ndarray, n_subvectors: int = 8, n_centroids: int = 256 ) -> Tuple[np.ndarray, dict]: """Product quantization for aggressive compression.""" from sklearn.cluster import KMeans n, dim = vectors.shape assert dim % n_subvectors == 0 subvector_dim = dim // n_subvectors codebooks = [] codes = np.zeros((n, n_subvectors), dtype=np.uint8) for i in range(n_subvectors): start = i * subvector_dim end = (i + 1) * subvector_dim subvectors = vectors[:, start:end] kmeans = KMeans(n_clusters=n_centroids, random_state=42) codes[:, i] = kmeans.fit_predict(subvectors) codebooks.append(kmeans.cluster_centers_) params = { "codebooks": codebooks, "n_subvectors": n_subvectors, "subvector_dim": subvector_dim } return codes, params @staticmethod def binary_quantize(vectors: np.ndarray) -> np.ndarray: """Binary quantization (sign of each dimension).""" # Convert to binary: positive = 1, negative = 0 binary = (vectors > 0).astype(np.uint8) # Pack bits into bytes n, dim = vectors.shape packed_dim = (dim + 7) // 8 packed = np.zeros((n, packed_dim), dtype=np.uint8) for i in range(dim): byte_idx = i // 8 bit_idx = i % 8 packed[:, byte_idx] |= (binary[:, i] << bit_idx) return packed def estimate_memory_usage( num_vectors: int, dimensions: int, quantization: str = "fp32", index_type: str = "hnsw", hnsw_m: int = 16 ) -> dict: """Estimate memory usage for different configurations.""" # Vector storage bytes_per_dimension = { "fp32": 4, "fp16": 2, "int8": 1, "pq": 0.05, # Approximate "binary": 0.125 } vector_bytes = num_vectors * dimensions * bytes_per_dimension[quantization] # Index overhead if index_type == "hnsw": # Each node has ~M*2 edges, each edge is 4 bytes (int32) index_bytes = num_vectors * hnsw_m * 2 * 4 elif index_type == "ivf": # Inverted lists + centroids index_bytes = num_vectors * 8 + 65536 * dimensions * 4 else: index_bytes = 0 total_bytes = vector_bytes + index_bytes return { "vector_storage_mb": vector_bytes / 1024 / 1024, "index_overhead_mb": index_bytes / 1024 / 1024, "total_mb": total_bytes / 1024 / 1024, "total_gb": total_bytes / 1024 / 1024 / 1024 } ``` ### Template 3: Qdrant Index Configuration ```python from qdrant_client import QdrantClient from qdrant_client.http import models def create_optimized_collection( client: QdrantClient, collection_name: str, vector_size: int, num_vectors: int, optimize_for: str = "balanced" # "recall", "speed", "memory" ) -> None: """Create collection with optimized settings.""" # HNSW configuration based on optimization target hnsw_configs = { "recall": models.HnswConfigDiff(m=32, ef_construct=256), "speed": models.HnswConfigDiff(m=16, ef_construct=64), "balanced": models.HnswConfigDiff(m=16, ef_construct=128), "memory": models.HnswConfigDiff(m=8, ef_construct=64) } # Quantization configuration quantization_configs = { "recall": None, # No quantization for max recall "speed": models.ScalarQuantization( scalar=models.ScalarQuantizationConfig( type=models.ScalarType.INT8, quantile=0.99, always_ram=True ) ), "balanced": models.ScalarQuantization( scalar=models.ScalarQuantizationConfig( type=models.ScalarType.INT8, quantile=0.99, always_ram=False ) ), "memory": models.ProductQuantization( product=models.ProductQuantizationConfig( compression=models.CompressionRatio.X16, always_ram=False ) ) } # Optimizer configuration optimizer_configs = { "recall": models.OptimizersConfigDiff( indexing_threshold=10000, memmap_threshold=50000 ), "speed": models.OptimizersConfigDiff( indexing_threshold=5000, memmap_threshold=20000 ), "balanced": models.OptimizersConfigDiff( indexing_threshold=20000, memmap_threshold=50000 ), "memory": models.OptimizersConfigDiff( indexing_threshold=50000, memmap_threshold=10000 # Use disk sooner ) } client.create_collection( collection_name=collection_name, vectors_config=models.VectorParams( size=vector_size, distance=models.Distance.COSINE ), hnsw_config=hnsw_configs[optimize_for], quantization_config=quantization_configs[optimize_for], optimizers_config=optimizer_configs[optimize_for] ) def tune_search_parameters( client: QdrantClient, collection_name: str, target_recall: float = 0.95 ) -> dict: """Tune search parameters for target recall.""" # Search parameter recommendations if target_recall >= 0.99: search_params = models.SearchParams( hnsw_ef=256, exact=False, quantization=models.QuantizationSearchParams( ignore=True, # Don't use quantization for search rescore=True ) ) elif target_recall >= 0.95: search_params = models.SearchParams( hnsw_ef=128, exact=False, quantization=models.QuantizationSearchParams( ignore=False, rescore=True, oversampling=2.0 ) ) else: search_params = models.SearchParams( hnsw_ef=64, exact=False, quantization=models.QuantizationSearchParams( ignore=False, rescore=False ) ) return search_params ``` ### Template 4: Performance Monitoring ```python import time from dataclasses import dataclass from typing import List import numpy as np @dataclass class SearchMetrics: latency_p50_ms: float latency_p95_ms: float latency_p99_ms: float recall: float qps: float class VectorSearchMonitor: """Monitor vector search performance.""" def __init__(self, ground_truth_fn=None): self.latencies = [] self.recalls = [] self.ground_truth_fn = ground_truth_fn def measure_search( self, search_fn, query_vectors: np.ndarray, k: int = 10, num_iterations: int = 100 ) -> SearchMetrics: """Benchmark search performance.""" latencies = [] for _ in range(num_iterations): for query in query_vectors: start = time.perf_counter() results = search_fn(query, k=k) latency = (time.perf_counter() - start) * 1000 latencies.append(latency) latencies = np.array(latencies) total_queries = num_iterations * len(query_vectors) total_time = sum(latencies) / 1000 # seconds return SearchMetrics( latency_p50_ms=np.percentile(latencies, 50), latency_p95_ms=np.percentile(latencies, 95), latency_p99_ms=np.percentile(latencies, 99), recall=self._calculate_recall(search_fn, query_vectors, k) if self.ground_truth_fn else 0, qps=total_queries / total_time ) def _calculate_recall(self, search_fn, queries: np.ndarray, k: int) -> float: """Calculate recall against ground truth.""" if not self.ground_truth_fn: return 0 correct = 0 total = 0 for query in queries: predicted = set(search_fn(query, k=k)) actual = set(self.ground_truth_fn(query, k=k)) correct += len(predicted & actual) total += k return correct / total def profile_index_build( build_fn, vectors: np.ndarray, batch_sizes: List[int] = [1000, 10000, 50000] ) -> dict: """Profile index build performance.""" results = {} for batch_size in batch_sizes: times = [] for i in range(0, len(vectors), batch_size): batch = vectors[i:i + batch_size] start = time.perf_counter() build_fn(batch) times.append(time.perf_counter() - start) results[batch_size] = { "avg_batch_time_s": np.mean(times), "vectors_per_second": batch_size / np.mean(times) } return results ``` ## Best Practices ### Do's - **Benchmark with real queries** - Synthetic may not represent production - **Monitor recall continuously** - Can degrade with data drift - **Start with defaults** - Tune only when needed - **Use quantization** - Significant memory savings - **Consider tiered storage** - Hot/cold data separation ### Don'ts - **Don't over-optimize early** - Profile first - **Don't ignore build time** - Index updates have cost - **Don't forget reindexing** - Plan for maintenance - **Don't skip warming** - Cold indexes are slow ## Resources - [HNSW Paper](https://arxiv.org/abs/1603.09320) - [Faiss Wiki](https://github.com/facebookresearch/faiss/wiki) - [ANN Benchmarks](https://ann-benchmarks.com/)
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

python-design-patterns

Python design patterns including KISS, Separation of Concerns,

coding
⭐1
# Python Design Patterns Write maintainable Python code using fundamental design principles. These patterns help you build systems that are easy to understand, test, and modify. ## When to Use This Skill - Designing new components or services - Refactoring complex or tangled code - Deciding whether to create an abstraction - Choosing between inheritance and composition - Evaluating code complexity and coupling - Planning modular architectures ## Core Concepts ### 1. KISS (Keep It Simple) Choose the simplest solution that works. Complexity must be justified by concrete requirements. ### 2. Single Responsibility (SRP) Each unit should have one reason to change. Separate concerns into focused components. ### 3. Composition Over Inheritance Build behavior by combining objects, not extending classes. ### 4. Rule of Three Wait until you have three instances before abstracting. Duplication is often better than premature abstraction. ## Quick Start ```python # Simple beats clever # Instead of a factory/registry pattern: FORMATTERS = {"json": JsonFormatter, "csv": CsvFormatter} def get_formatter(name: str) -> Formatter: return FORMATTERS[name]() ``` ## Fundamental Patterns ### Pattern 1: KISS - Keep It Simple Before adding complexity, ask: does a simpler solution work? ```python # Over-engineered: Factory with registration class OutputFormatterFactory: _formatters: dict[str, type[Formatter]] = {} @classmethod def register(cls, name: str): def decorator(formatter_cls): cls._formatters[name] = formatter_cls return formatter_cls return decorator @classmethod def create(cls, name: str) -> Formatter: return cls._formatters[name]() @OutputFormatterFactory.register("json") class JsonFormatter(Formatter): ... # Simple: Just use a dictionary FORMATTERS = { "json": JsonFormatter, "csv": CsvFormatter, "xml": XmlFormatter, } def get_formatter(name: str) -> Formatter: """Get formatter by name.""" if name not in FORMATTERS: raise ValueError(f"Unknown format: {name}") return FORMATTERS[name]() ``` The factory pattern adds code without adding value here. Save patterns for when they solve real problems. ### Pattern 2: Single Responsibility Principle Each class or function should have one reason to change. ```python # BAD: Handler does everything class UserHandler: async def create_user(self, request: Request) -> Response: # HTTP parsing data = await request.json() # Validation if not data.get("email"): return Response({"error": "email required"}, status=400) # Database access user = await db.execute( "INSERT INTO users (email, name) VALUES ($1, $2) RETURNING *", data["email"], data["name"] ) # Response formatting return Response({"id": user.id, "email": user.email}, status=201) # GOOD: Separated concerns class UserService: """Business logic only.""" def __init__(self, repo: UserRepository) -> None: self._repo = repo async def create_user(self, data: CreateUserInput) -> User: # Only business rules here user = User(email=data.email, name=data.name) return await self._repo.save(user) class UserHandler: """HTTP concerns only.""" def __init__(self, service: UserService) -> None: self._service = service async def create_user(self, request: Request) -> Response: data = CreateUserInput(**(await request.json())) user = await self._service.create_user(data) return Response(user.to_dict(), status=201) ``` Now HTTP changes don't affect business logic, and vice versa. ### Pattern 3: Separation of Concerns Organize code into distinct layers with clear responsibilities. ``` ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ API Layer (handlers) │ │ - Parse requests │ │ - Call services │ │ - Format responses │ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ │ ā–¼ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ Service Layer (business logic) │ │ - Domain rules and validation │ │ - Orchestrate operations │ │ - Pure functions where possible │ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ │ ā–¼ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ Repository Layer (data access) │ │ - SQL queries │ │ - External API calls │ │ - Cache operations │ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ``` Each layer depends only on layers below it: ```python # Repository: Data access class UserRepository: async def get_by_id(self, user_id: str) -> User | None: row = await self._db.fetchrow( "SELECT * FROM users WHERE id = $1", user_id ) return User(**row) if row else None # Service: Business logic class UserService: def __init__(self, repo: UserRepository) -> None: self._repo = repo async def get_user(self, user_id: str) -> User: user = await self._repo.get_by_id(user_id) if user is None: raise UserNotFoundError(user_id) return user # Handler: HTTP concerns @app.get("/users/{user_id}") async def get_user(user_id: str) -> UserResponse: user = await user_service.get_user(user_id) return UserResponse.from_user(user) ``` ### Pattern 4: Composition Over Inheritance Build behavior by combining objects rather than inheriting. ```python # Inheritance: Rigid and hard to test class EmailNotificationService(NotificationService): def __init__(self): super().__init__() self._smtp = SmtpClient() # Hard to mock def notify(self, user: User, message: str) -> None: self._smtp.send(user.email, message) # Composition: Flexible and testable class NotificationService: """Send notifications via multiple channels.""" def __init__( self, email_sender: EmailSender, sms_sender: SmsSender | None = None, push_sender: PushSender | None = None, ) -> None: self._email = email_sender self._sms = sms_sender self._push = push_sender async def notify( self, user: User, message: str, channels: set[str] | None = None, ) -> None: channels = channels or {"email"} if "email" in channels: await self._email.send(user.email, message) if "sms" in channels and self._sms and user.phone: await self._sms.send(user.phone, message) if "push" in channels and self._push and user.device_token: await self._push.send(user.device_token, message) # Easy to test with fakes service = NotificationService( email_sender=FakeEmailSender(), sms_sender=FakeSmsSender(), ) ``` ## Advanced Patterns ### Pattern 5: Rule of Three Wait until you have three instances before abstracting. ```python # Two similar functions? Don't abstract yet def process_orders(orders: list[Order]) -> list[Result]: results = [] for order in orders: validated = validate_order(order) result = process_validated_order(validated) results.append(result) return results def process_returns(returns: list[Return]) -> list[Result]: results = [] for ret in returns: validated = validate_return(ret) result = process_validated_return(validated) results.append(result) return results # These look similar, but wait! Are they actually the same? # Different validation, different processing, different errors... # Duplication is often better than the wrong abstraction # Only after a third case, consider if there's a real pattern # But even then, sometimes explicit is better than abstract ``` ### Pattern 6: Function Size Guidelines Keep functions focused. Extract when a function: - Exceeds 20-50 lines (varies by complexity) - Serves multiple distinct purposes - Has deeply nested logic (3+ levels) ```python # Too long, multiple concerns mixed def process_order(order: Order) -> Result: # 50 lines of validation... # 30 lines of inventory check... # 40 lines of payment processing... # 20 lines of notification... pass # Better: Composed from focused functions def process_order(order: Order) -> Result: """Process a customer order through the complete workflow.""" validate_order(order) reserve_inventory(order) payment_result = charge_payment(order) send_confirmation(order, payment_result) return Result(success=True, order_id=order.id) ``` ### Pattern 7: Dependency Injection Pass dependencies through constructors for testability. ```python from typing import Protocol class Logger(Protocol): def info(self, msg: str, **kwargs) -> None: ... def error(self, msg: str, **kwargs) -> None: ... class Cache(Protocol): async def get(self, key: str) -> str | None: ... async def set(self, key: str, value: str, ttl: int) -> None: ... class UserService: """Service with injected dependencies.""" def __init__( self, repository: UserRepository, cache: Cache, logger: Logger, ) -> None: self._repo = repository self._cache = cache self._logger = logger async def get_user(self, user_id: str) -> User: # Check cache first cached = await self._cache.get(f"user:{user_id}") if cached: self._logger.info("Cache hit", user_id=user_id) return User.from_json(cached) # Fetch from database user = await self._repo.get_by_id(user_id) if user: await self._cache.set(f"user:{user_id}", user.to_json(), ttl=300) return user # Production service = UserService( repository=PostgresUserRepository(db), cache=RedisCache(redis), logger=StructlogLogger(), ) # Testing service = UserService( repository=InMemoryUserRepository(), cache=FakeCache(), logger=NullLogger(), ) ``` ### Pattern 8: Avoiding Common Anti-Patterns **Don't expose internal types:** ```python # BAD: Leaking ORM model to API @app.get("/users/{id}") def get_user(id: str) -> UserModel: # SQLAlchemy model return db.query(UserModel).get(id) # GOOD: Use response schemas @app.get("/users/{id}") def get_user(id: str) -> UserResponse: user = db.query(UserModel).get(id) return UserResponse.from_orm(user) ``` **Don't mix I/O with business logic:** ```python # BAD: SQL embedded in business logic def calculate_discount(user_id: str) -> float: user = db.query("SELECT * FROM users WHERE id = ?", user_id) orders = db.query("SELECT * FROM orders WHERE user_id = ?", user_id) # Business logic mixed with data access # GOOD: Repository pattern def calculate_discount(user: User, order_history: list[Order]) -> float: # Pure business logic, easily testable if len(order_history) > 10: return 0.15 return 0.0 ``` ## Best Practices Summary 1. **Keep it simple** - Choose the simplest solution that works 2. **Single responsibility** - Each unit has one reason to change 3. **Separate concerns** - Distinct layers with clear purposes 4. **Compose, don't inherit** - Combine objects for flexibility 5. **Rule of three** - Wait before abstracting 6. **Keep functions small** - 20-50 lines (varies by complexity), one purpose 7. **Inject dependencies** - Constructor injection for testability 8. **Delete before abstracting** - Remove dead code, then consider patterns 9. **Test each layer** - Isolated tests for each concern 10. **Explicit over clever** - Readable code beats elegant code
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

python-resilience

Python resilience patterns including automatic retries, exponential

coding
⭐1
# Python Resilience Patterns Build fault-tolerant Python applications that gracefully handle transient failures, network issues, and service outages. Resilience patterns keep systems running when dependencies are unreliable. ## When to Use This Skill - Adding retry logic to external service calls - Implementing timeouts for network operations - Building fault-tolerant microservices - Handling rate limiting and backpressure - Creating infrastructure decorators - Designing circuit breakers ## Core Concepts ### 1. Transient vs Permanent Failures Retry transient errors (network timeouts, temporary service issues). Don't retry permanent errors (invalid credentials, bad requests). ### 2. Exponential Backoff Increase wait time between retries to avoid overwhelming recovering services. ### 3. Jitter Add randomness to backoff to prevent thundering herd when many clients retry simultaneously. ### 4. Bounded Retries Cap both attempt count and total duration to prevent infinite retry loops. ## Quick Start ```python from tenacity import retry, stop_after_attempt, wait_exponential_jitter @retry( stop=stop_after_attempt(3), wait=wait_exponential_jitter(initial=1, max=10), ) def call_external_service(request: dict) -> dict: return httpx.post("https://api.example.com", json=request).json() ``` ## Fundamental Patterns ### Pattern 1: Basic Retry with Tenacity Use the `tenacity` library for production-grade retry logic. For simpler cases, consider built-in retry functionality or a lightweight custom implementation. ```python from tenacity import ( retry, stop_after_attempt, stop_after_delay, wait_exponential_jitter, retry_if_exception_type, ) TRANSIENT_ERRORS = (ConnectionError, TimeoutError, OSError) @retry( retry=retry_if_exception_type(TRANSIENT_ERRORS), stop=stop_after_attempt(5) | stop_after_delay(60), wait=wait_exponential_jitter(initial=1, max=30), ) def fetch_data(url: str) -> dict: """Fetch data with automatic retry on transient failures.""" response = httpx.get(url, timeout=30) response.raise_for_status() return response.json() ``` ### Pattern 2: Retry Only Appropriate Errors Whitelist specific transient exceptions. Never retry: - `ValueError`, `TypeError` - These are bugs, not transient issues - `AuthenticationError` - Invalid credentials won't become valid - HTTP 4xx errors (except 429) - Client errors are permanent ```python from tenacity import retry, retry_if_exception_type import httpx # Define what's retryable RETRYABLE_EXCEPTIONS = ( ConnectionError, TimeoutError, httpx.ConnectTimeout, httpx.ReadTimeout, ) @retry( retry=retry_if_exception_type(RETRYABLE_EXCEPTIONS), stop=stop_after_attempt(3), wait=wait_exponential_jitter(initial=1, max=10), ) def resilient_api_call(endpoint: str) -> dict: """Make API call with retry on network issues.""" return httpx.get(endpoint, timeout=10).json() ``` ### Pattern 3: HTTP Status Code Retries Retry specific HTTP status codes that indicate transient issues. ```python from tenacity import retry, retry_if_result, stop_after_attempt import httpx RETRY_STATUS_CODES = {429, 502, 503, 504} def should_retry_response(response: httpx.Response) -> bool: """Check if response indicates a retryable error.""" return response.status_code in RETRY_STATUS_CODES @retry( retry=retry_if_result(should_retry_response), stop=stop_after_attempt(3), wait=wait_exponential_jitter(initial=1, max=10), ) def http_request(method: str, url: str, **kwargs) -> httpx.Response: """Make HTTP request with retry on transient status codes.""" return httpx.request(method, url, timeout=30, **kwargs) ``` ### Pattern 4: Combined Exception and Status Retry Handle both network exceptions and HTTP status codes. ```python from tenacity import ( retry, retry_if_exception_type, retry_if_result, stop_after_attempt, wait_exponential_jitter, before_sleep_log, ) import logging import httpx logger = logging.getLogger(__name__) TRANSIENT_EXCEPTIONS = ( ConnectionError, TimeoutError, httpx.ConnectError, httpx.ReadTimeout, ) RETRY_STATUS_CODES = {429, 500, 502, 503, 504} def is_retryable_response(response: httpx.Response) -> bool: return response.status_code in RETRY_STATUS_CODES @retry( retry=( retry_if_exception_type(TRANSIENT_EXCEPTIONS) | retry_if_result(is_retryable_response) ), stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=1, max=30), before_sleep=before_sleep_log(logger, logging.WARNING), ) def robust_http_call( method: str, url: str, **kwargs, ) -> httpx.Response: """HTTP call with comprehensive retry handling.""" return httpx.request(method, url, timeout=30, **kwargs) ``` ## Advanced Patterns ### Pattern 5: Logging Retry Attempts Track retry behavior for debugging and alerting. ```python from tenacity import retry, stop_after_attempt, wait_exponential import structlog logger = structlog.get_logger() def log_retry_attempt(retry_state): """Log detailed retry information.""" exception = retry_state.outcome.exception() logger.warning( "Retrying operation", attempt=retry_state.attempt_number, exception_type=type(exception).__name__, exception_message=str(exception), next_wait_seconds=retry_state.next_action.sleep if retry_state.next_action else None, ) @retry( stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, max=10), before_sleep=log_retry_attempt, ) def call_with_logging(request: dict) -> dict: """External call with retry logging.""" ... ``` ### Pattern 6: Timeout Decorator Create reusable timeout decorators for consistent timeout handling. ```python import asyncio from functools import wraps from typing import TypeVar, Callable T = TypeVar("T") def with_timeout(seconds: float): """Decorator to add timeout to async functions.""" def decorator(func: Callable[..., T]) -> Callable[..., T]: @wraps(func) async def wrapper(*args, **kwargs) -> T: return await asyncio.wait_for( func(*args, **kwargs), timeout=seconds, ) return wrapper return decorator @with_timeout(30) async def fetch_with_timeout(url: str) -> dict: """Fetch URL with 30 second timeout.""" async with httpx.AsyncClient() as client: response = await client.get(url) return response.json() ``` ### Pattern 7: Cross-Cutting Concerns via Decorators Stack decorators to separate infrastructure from business logic. ```python from functools import wraps from typing import TypeVar, Callable import structlog logger = structlog.get_logger() T = TypeVar("T") def traced(name: str | None = None): """Add tracing to function calls.""" def decorator(func: Callable[..., T]) -> Callable[..., T]: span_name = name or func.__name__ @wraps(func) async def wrapper(*args, **kwargs) -> T: logger.info("Operation started", operation=span_name) try: result = await func(*args, **kwargs) logger.info("Operation completed", operation=span_name) return result except Exception as e: logger.error("Operation failed", operation=span_name, error=str(e)) raise return wrapper return decorator # Stack multiple concerns @traced("fetch_user_data") @with_timeout(30) @retry(stop=stop_after_attempt(3), wait=wait_exponential_jitter()) async def fetch_user_data(user_id: str) -> dict: """Fetch user with tracing, timeout, and retry.""" ... ``` ### Pattern 8: Dependency Injection for Testability Pass infrastructure components through constructors for easy testing. ```python from dataclasses import dataclass from typing import Protocol class Logger(Protocol): def info(self, msg: str, **kwargs) -> None: ... def error(self, msg: str, **kwargs) -> None: ... class MetricsClient(Protocol): def increment(self, metric: str, tags: dict | None = None) -> None: ... def timing(self, metric: str, value: float) -> None: ... @dataclass class UserService: """Service with injected infrastructure.""" repository: UserRepository logger: Logger metrics: MetricsClient async def get_user(self, user_id: str) -> User: self.logger.info("Fetching user", user_id=user_id) start = time.perf_counter() try: user = await self.repository.get(user_id) self.metrics.increment("user.fetch.success") return user except Exception as e: self.metrics.increment("user.fetch.error") self.logger.error("Failed to fetch user", user_id=user_id, error=str(e)) raise finally: elapsed = time.perf_counter() - start self.metrics.timing("user.fetch.duration", elapsed) # Easy to test with fakes service = UserService( repository=FakeRepository(), logger=FakeLogger(), metrics=FakeMetrics(), ) ``` ### Pattern 9: Fail-Safe Defaults Degrade gracefully when non-critical operations fail. ```python from typing import TypeVar from collections.abc import Callable T = TypeVar("T") def fail_safe(default: T, log_failure: bool = True): """Return default value on failure instead of raising.""" def decorator(func: Callable[..., T]) -> Callable[..., T]: @wraps(func) async def wrapper(*args, **kwargs) -> T: try: return await func(*args, **kwargs) except Exception as e: if log_failure: logger.warning( "Operation failed, using default", function=func.__name__, error=str(e), ) return default return wrapper return decorator @fail_safe(default=[]) async def get_recommendations(user_id: str) -> list[str]: """Get recommendations, return empty list on failure.""" ... ``` ## Best Practices Summary 1. **Retry only transient errors** - Don't retry bugs or authentication failures 2. **Use exponential backoff** - Give services time to recover 3. **Add jitter** - Prevent thundering herd from synchronized retries 4. **Cap total duration** - `stop_after_attempt(5) | stop_after_delay(60)` 5. **Log every retry** - Silent retries hide systemic problems 6. **Use decorators** - Keep retry logic separate from business logic 7. **Inject dependencies** - Make infrastructure testable 8. **Set timeouts everywhere** - Every network call needs a timeout 9. **Fail gracefully** - Return cached/default values for non-critical paths 10. **Monitor retry rates** - High retry rates indicate underlying issues
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

attack-tree-construction

Build comprehensive attack trees to visualize threat paths. Use

security
⭐1
# Attack Tree Construction Systematic attack path visualization and analysis. ## When to Use This Skill - Visualizing complex attack scenarios - Identifying defense gaps and priorities - Communicating risks to stakeholders - Planning defensive investments - Penetration test planning - Security architecture review ## Core Concepts ### 1. Attack Tree Structure ``` [Root Goal] | ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ │ [Sub-goal 1] [Sub-goal 2] (OR node) (AND node) │ │ ā”Œā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā” │ │ │ │ [Attack] [Attack] [Attack] [Attack] (leaf) (leaf) (leaf) (leaf) ``` ### 2. Node Types | Type | Symbol | Description | | -------- | --------- | ----------------------- | | **OR** | Oval | Any child achieves goal | | **AND** | Rectangle | All children required | | **Leaf** | Box | Atomic attack step | ### 3. Attack Attributes | Attribute | Description | Values | | ------------- | ----------------------- | ------------------ | | **Cost** | Resources needed | $, $$, $$$ | | **Time** | Duration to execute | Hours, Days, Weeks | | **Skill** | Expertise required | Low, Medium, High | | **Detection** | Likelihood of detection | Low, Medium, High | ## Templates ### Template 1: Attack Tree Data Model ```python from dataclasses import dataclass, field from enum import Enum from typing import List, Dict, Optional, Union import json class NodeType(Enum): OR = "or" AND = "and" LEAF = "leaf" class Difficulty(Enum): TRIVIAL = 1 LOW = 2 MEDIUM = 3 HIGH = 4 EXPERT = 5 class Cost(Enum): FREE = 0 LOW = 1 MEDIUM = 2 HIGH = 3 VERY_HIGH = 4 class DetectionRisk(Enum): NONE = 0 LOW = 1 MEDIUM = 2 HIGH = 3 CERTAIN = 4 @dataclass class AttackAttributes: difficulty: Difficulty = Difficulty.MEDIUM cost: Cost = Cost.MEDIUM detection_risk: DetectionRisk = DetectionRisk.MEDIUM time_hours: float = 8.0 requires_insider: bool = False requires_physical: bool = False @dataclass class AttackNode: id: str name: str description: str node_type: NodeType attributes: AttackAttributes = field(default_factory=AttackAttributes) children: List['AttackNode'] = field(default_factory=list) mitigations: List[str] = field(default_factory=list) cve_refs: List[str] = field(default_factory=list) def add_child(self, child: 'AttackNode') -> None: self.children.append(child) def calculate_path_difficulty(self) -> float: """Calculate aggregate difficulty for this path.""" if self.node_type == NodeType.LEAF: return self.attributes.difficulty.value if not self.children: return 0 child_difficulties = [c.calculate_path_difficulty() for c in self.children] if self.node_type == NodeType.OR: return min(child_difficulties) else: # AND return max(child_difficulties) def calculate_path_cost(self) -> float: """Calculate aggregate cost for this path.""" if self.node_type == NodeType.LEAF: return self.attributes.cost.value if not self.children: return 0 child_costs = [c.calculate_path_cost() for c in self.children] if self.node_type == NodeType.OR: return min(child_costs) else: # AND return sum(child_costs) def to_dict(self) -> Dict: """Convert to dictionary for serialization.""" return { "id": self.id, "name": self.name, "description": self.description, "type": self.node_type.value, "attributes": { "difficulty": self.attributes.difficulty.name, "cost": self.attributes.cost.name, "detection_risk": self.attributes.detection_risk.name, "time_hours": self.attributes.time_hours, }, "mitigations": self.mitigations, "children": [c.to_dict() for c in self.children] } @dataclass class AttackTree: name: str description: str root: AttackNode version: str = "1.0" def find_easiest_path(self) -> List[AttackNode]: """Find the path with lowest difficulty.""" return self._find_path(self.root, minimize="difficulty") def find_cheapest_path(self) -> List[AttackNode]: """Find the path with lowest cost.""" return self._find_path(self.root, minimize="cost") def find_stealthiest_path(self) -> List[AttackNode]: """Find the path with lowest detection risk.""" return self._find_path(self.root, minimize="detection") def _find_path( self, node: AttackNode, minimize: str ) -> List[AttackNode]: """Recursive path finding.""" if node.node_type == NodeType.LEAF: return [node] if not node.children: return [node] if node.node_type == NodeType.OR: # Pick the best child path best_path = None best_score = float('inf') for child in node.children: child_path = self._find_path(child, minimize) score = self._path_score(child_path, minimize) if score < best_score: best_score = score best_path = child_path return [node] + (best_path or []) else: # AND # Must traverse all children path = [node] for child in node.children: path.extend(self._find_path(child, minimize)) return path def _path_score(self, path: List[AttackNode], metric: str) -> float: """Calculate score for a path.""" if metric == "difficulty": return sum(n.attributes.difficulty.value for n in path if n.node_type == NodeType.LEAF) elif metric == "cost": return sum(n.attributes.cost.value for n in path if n.node_type == NodeType.LEAF) elif metric == "detection": return sum(n.attributes.detection_risk.value for n in path if n.node_type == NodeType.LEAF) return 0 def get_all_leaf_attacks(self) -> List[AttackNode]: """Get all leaf attack nodes.""" leaves = [] self._collect_leaves(self.root, leaves) return leaves def _collect_leaves(self, node: AttackNode, leaves: List[AttackNode]) -> None: if node.node_type == NodeType.LEAF: leaves.append(node) for child in node.children: self._collect_leaves(child, leaves) def get_unmitigated_attacks(self) -> List[AttackNode]: """Find attacks without mitigations.""" return [n for n in self.get_all_leaf_attacks() if not n.mitigations] def export_json(self) -> str: """Export tree to JSON.""" return json.dumps({ "name": self.name, "description": self.description, "version": self.version, "root": self.root.to_dict() }, indent=2) ``` ### Template 2: Attack Tree Builder ```python class AttackTreeBuilder: """Fluent builder for attack trees.""" def __init__(self, name: str, description: str): self.name = name self.description = description self._node_stack: List[AttackNode] = [] self._root: Optional[AttackNode] = None def goal(self, id: str, name: str, description: str = "") -> 'AttackTreeBuilder': """Set the root goal (OR node by default).""" self._root = AttackNode( id=id, name=name, description=description, node_type=NodeType.OR ) self._node_stack = [self._root] return self def or_node(self, id: str, name: str, description: str = "") -> 'AttackTreeBuilder': """Add an OR sub-goal.""" node = AttackNode( id=id, name=name, description=description, node_type=NodeType.OR ) self._current().add_child(node) self._node_stack.append(node) return self def and_node(self, id: str, name: str, description: str = "") -> 'AttackTreeBuilder': """Add an AND sub-goal (all children required).""" node = AttackNode( id=id, name=name, description=description, node_type=NodeType.AND ) self._current().add_child(node) self._node_stack.append(node) return self def attack( self, id: str, name: str, description: str = "", difficulty: Difficulty = Difficulty.MEDIUM, cost: Cost = Cost.MEDIUM, detection: DetectionRisk = DetectionRisk.MEDIUM, time_hours: float = 8.0, mitigations: List[str] = None ) -> 'AttackTreeBuilder': """Add a leaf attack node.""" node = AttackNode( id=id, name=name, description=description, node_type=NodeType.LEAF, attributes=AttackAttributes( difficulty=difficulty, cost=cost, detection_risk=detection, time_hours=time_hours ), mitigations=mitigations or [] ) self._current().add_child(node) return self def end(self) -> 'AttackTreeBuilder': """Close current node, return to parent.""" if len(self._node_stack) > 1: self._node_stack.pop() return self def build(self) -> AttackTree: """Build the attack tree.""" if not self._root: raise ValueError("No root goal defined") return AttackTree( name=self.name, description=self.description, root=self._root ) def _current(self) -> AttackNode: if not self._node_stack: raise ValueError("No current node") return self._node_stack[-1] # Example usage def build_account_takeover_tree() -> AttackTree: """Build attack tree for account takeover scenario.""" return ( AttackTreeBuilder("Account Takeover", "Gain unauthorized access to user account") .goal("G1", "Take Over User Account") .or_node("S1", "Steal Credentials") .attack( "A1", "Phishing Attack", difficulty=Difficulty.LOW, cost=Cost.LOW, detection=DetectionRisk.MEDIUM, mitigations=["Security awareness training", "Email filtering"] ) .attack( "A2", "Credential Stuffing", difficulty=Difficulty.TRIVIAL, cost=Cost.LOW, detection=DetectionRisk.HIGH, mitigations=["Rate limiting", "MFA", "Password breach monitoring"] ) .attack( "A3", "Keylogger Malware", difficulty=Difficulty.MEDIUM, cost=Cost.MEDIUM, detection=DetectionRisk.MEDIUM, mitigations=["Endpoint protection", "MFA"] ) .end() .or_node("S2", "Bypass Authentication") .attack( "A4", "Session Hijacking", difficulty=Difficulty.MEDIUM, cost=Cost.LOW, detection=DetectionRisk.LOW, mitigations=["Secure session management", "HTTPS only"] ) .attack( "A5", "Authentication Bypass Vulnerability", difficulty=Difficulty.HIGH, cost=Cost.LOW, detection=DetectionRisk.LOW, mitigations=["Security testing", "Code review", "WAF"] ) .end() .or_node("S3", "Social Engineering") .and_node("S3.1", "Account Recovery Attack") .attack( "A6", "Gather Personal Information", difficulty=Difficulty.LOW, cost=Cost.FREE, detection=DetectionRisk.NONE ) .attack( "A7", "Call Support Desk", difficulty=Difficulty.MEDIUM, cost=Cost.FREE, detection=DetectionRisk.MEDIUM, mitigations=["Support verification procedures", "Security questions"] ) .end() .end() .build() ) ``` ### Template 3: Mermaid Diagram Generator ```python class MermaidExporter: """Export attack trees to Mermaid diagram format.""" def __init__(self, tree: AttackTree): self.tree = tree self._lines: List[str] = [] self._node_count = 0 def export(self) -> str: """Export tree to Mermaid flowchart.""" self._lines = ["flowchart TD"] self._export_node(self.tree.root, None) return "\n".join(self._lines) def _export_node(self, node: AttackNode, parent_id: Optional[str]) -> str: """Recursively export nodes.""" node_id = f"N{self._node_count}" self._node_count += 1 # Node shape based on type if node.node_type == NodeType.OR: shape = f"{node_id}(({node.name}))" elif node.node_type == NodeType.AND: shape = f"{node_id}[{node.name}]" else: # LEAF # Color based on difficulty style = self._get_leaf_style(node) shape = f"{node_id}[/{node.name}/]" self._lines.append(f" style {node_id} {style}") self._lines.append(f" {shape}") if parent_id: connector = "-->" if node.node_type != NodeType.AND else "==>" self._lines.append(f" {parent_id} {connector} {node_id}") for child in node.children: self._export_node(child, node_id) return node_id def _get_leaf_style(self, node: AttackNode) -> str: """Get style based on attack attributes.""" colors = { Difficulty.TRIVIAL: "fill:#ff6b6b", # Red - easy attack Difficulty.LOW: "fill:#ffa06b", Difficulty.MEDIUM: "fill:#ffd93d", Difficulty.HIGH: "fill:#6bcb77", Difficulty.EXPERT: "fill:#4d96ff", # Blue - hard attack } color = colors.get(node.attributes.difficulty, "fill:#gray") return color class PlantUMLExporter: """Export attack trees to PlantUML format.""" def __init__(self, tree: AttackTree): self.tree = tree def export(self) -> str: """Export tree to PlantUML.""" lines = [ "@startmindmap", f"* {self.tree.name}", ] self._export_node(self.tree.root, lines, 1) lines.append("@endmindmap") return "\n".join(lines) def _export_node(self, node: AttackNode, lines: List[str], depth: int) -> None: """Recursively export nodes.""" prefix = "*" * (depth + 1) if node.node_type == NodeType.OR: marker = "[OR]" elif node.node_type == NodeType.AND: marker = "[AND]" else: diff = node.attributes.difficulty.name marker = f"<<{diff}>>" lines.append(f"{prefix} {marker} {node.name}") for child in node.children: self._export_node(child, lines, depth + 1) ``` ### Template 4: Attack Path Analysis ```python from typing import Set, Tuple class AttackPathAnalyzer: """Analyze attack paths and coverage.""" def __init__(self, tree: AttackTree): self.tree = tree def get_all_paths(self) -> List[List[AttackNode]]: """Get all possible attack paths.""" paths = [] self._collect_paths(self.tree.root, [], paths) return paths def _collect_paths( self, node: AttackNode, current_path: List[AttackNode], all_paths: List[List[AttackNode]] ) -> None: """Recursively collect all paths.""" current_path = current_path + [node] if node.node_type == NodeType.LEAF: all_paths.append(current_path) return if not node.children: all_paths.append(current_path) return if node.node_type == NodeType.OR: # Each child is a separate path for child in node.children: self._collect_paths(child, current_path, all_paths) else: # AND # Must combine all children child_paths = [] for child in node.children: child_sub_paths = [] self._collect_paths(child, [], child_sub_paths) child_paths.append(child_sub_paths) # Combine paths from all AND children combined = self._combine_and_paths(child_paths) for combo in combined: all_paths.append(current_path + combo) def _combine_and_paths( self, child_paths: List[List[List[AttackNode]]] ) -> List[List[AttackNode]]: """Combine paths from AND node children.""" if not child_paths: return [[]] if len(child_paths) == 1: return [path for paths in child_paths for path in paths] # Cartesian product of all child path combinations result = [[]] for paths in child_paths: new_result = [] for existing in result: for path in paths: new_result.append(existing + path) result = new_result return result def calculate_path_metrics(self, path: List[AttackNode]) -> Dict: """Calculate metrics for a specific path.""" leaves = [n for n in path if n.node_type == NodeType.LEAF] total_difficulty = sum(n.attributes.difficulty.value for n in leaves) total_cost = sum(n.attributes.cost.value for n in leaves) total_time = sum(n.attributes.time_hours for n in leaves) max_detection = max((n.attributes.detection_risk.value for n in leaves), default=0) return { "steps": len(leaves), "total_difficulty": total_difficulty, "avg_difficulty": total_difficulty / len(leaves) if leaves else 0, "total_cost": total_cost, "total_time_hours": total_time, "max_detection_risk": max_detection, "requires_insider": any(n.attributes.requires_insider for n in leaves), "requires_physical": any(n.attributes.requires_physical for n in leaves), } def identify_critical_nodes(self) -> List[Tuple[AttackNode, int]]: """Find nodes that appear in the most paths.""" paths = self.get_all_paths() node_counts: Dict[str, Tuple[AttackNode, int]] = {} for path in paths: for node in path: if node.id not in node_counts: node_counts[node.id] = (node, 0) node_counts[node.id] = (node, node_counts[node.id][1] + 1) return sort
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

competitive-landscape

This skill should be used when the user asks to "analyze

business
⭐1
# Competitive Landscape Analysis Comprehensive frameworks for analyzing competition, identifying differentiation opportunities, and developing winning market positioning strategies. ## Overview Understand competitive dynamics using proven frameworks (Porter's Five Forces, Blue Ocean Strategy, positioning maps) to identify opportunities and craft defensible competitive advantages. ## Porter's Five Forces Analyze industry attractiveness and competitive intensity. ### Force 1: Threat of New Entrants **Barriers to Entry:** - Capital requirements - Economies of scale - Switching costs - Brand loyalty - Regulatory barriers - Access to distribution - Network effects **High Threat:** Low barriers, easy to enter (e.g., simple SaaS tools) **Low Threat:** High barriers (e.g., regulated industries, hardware) **Analysis Questions:** - How easy is it for new competitors to enter? - What would it cost to launch a competing product? - Are there network effects or switching costs protecting incumbents? ### Force 2: Bargaining Power of Suppliers **Supplier Power Factors:** - Supplier concentration - Availability of substitutes - Importance to supplier - Switching costs - Forward integration threat **High Power:** Few suppliers, critical inputs (e.g., cloud infrastructure providers) **Low Power:** Many alternatives, commoditized (e.g., generic services) **Analysis Questions:** - Who are our critical suppliers? - Could they raise prices or reduce quality? - Can we switch suppliers easily? ### Force 3: Bargaining Power of Buyers **Buyer Power Factors:** - Buyer concentration - Volume purchased - Product differentiation - Price sensitivity - Backward integration threat **High Power:** Few large customers, standardized products (e.g., enterprise deals) **Low Power:** Many small customers, differentiated product (e.g., consumer subscriptions) **Analysis Questions:** - Can customers easily switch to competitors? - Do few customers generate most revenue? - How price-sensitive are buyers? ### Force 4: Threat of Substitutes **Substitute Considerations:** - Alternative solutions - Price-performance tradeoff - Switching costs - Buyer propensity to substitute **High Threat:** Many alternatives, low switching cost (e.g., productivity software) **Low Threat:** Unique solution, high switching cost (e.g., ERP systems) **Analysis Questions:** - What alternative ways can customers solve this problem? - How do substitutes compare on price and performance? - What's the cost to switch to a substitute? ### Force 5: Competitive Rivalry **Rivalry Intensity Factors:** - Number of competitors - Industry growth rate - Product differentiation - Exit barriers - Strategic stakes **High Rivalry:** Many competitors, slow growth, commoditized (e.g., email marketing) **Low Rivalry:** Few competitors, fast growth, differentiated (e.g., emerging AI tools) **Analysis Questions:** - How many direct competitors exist? - Is the market growing or stagnant? - How differentiated are offerings? - Are competitors competing on price or value? ### Forces Analysis Summary Create a scorecard: | Force | Intensity (1-5) | Impact | Key Factors | | -------------- | --------------- | ------ | --------------------------------- | | New Entrants | 3 | Medium | Low barriers but network effects | | Supplier Power | 2 | Low | Many cloud providers | | Buyer Power | 4 | High | Enterprise customers concentrated | | Substitutes | 3 | Medium | Manual processes alternative | | Rivalry | 4 | High | 10+ direct competitors | **Overall Assessment:** Moderate industry attractiveness with high rivalry and buyer power ## Blue Ocean Strategy Identify uncontested market space through value innovation. ### Four Actions Framework **Eliminate:** What factors can be eliminated that the industry takes for granted? **Reduce:** What factors can be reduced well below industry standard? **Raise:** What factors can be raised well above industry standard? **Create:** What factors can be created that the industry never offered? ### Strategy Canvas Map your offering vs. competitors on key factors. **Example: Budget Hotels** ``` High | ā˜… Traditional Hotels | ā˜… Budget Hotels (new) | Low |___________________________________ Price Luxury Convenience Cleanliness Budget Hotel Strategy: - Eliminate: Luxury amenities, room service - Reduce: Lobby size, staff - Raise: Cleanliness, online booking - Create: Self-service kiosks, mobile app ``` ### Value Innovation Find the sweet spot: Lower cost + higher value **Steps:** 1. Map industry competing factors 2. Identify factors to eliminate/reduce (cost savings) 3. Identify factors to raise/create (differentiation) 4. Validate that combination creates new market space ## Competitive Positioning ### Positioning Map Plot competitors on 2-3 key dimensions. **Example Dimensions:** - Price vs. Features - Complexity vs. Ease of Use - Enterprise vs. SMB Focus - Self-Service vs. High-Touch - Generalist vs. Specialist **How to Create:** 1. Choose 2 dimensions most important to customers 2. Plot all competitors 3. Identify gaps (white space) 4. Validate gap represents real customer need **Example:** ``` High Price | | ā˜… Enterprise A ā˜… Enterprise B | | ā— Our Position (gap) | | ā˜… Competitor C ā˜… Competitor D | Low Price |____________________________________________ Simple Complex ``` ### Differentiation Strategy **How to Differentiate:** 1. **Product Differentiation** - Unique features - Superior performance - Better design/UX - Integration ecosystem 2. **Service Differentiation** - Customer support quality - Onboarding experience - Response time - Success programs 3. **Brand Differentiation** - Trust and reputation - Thought leadership - Community - Values alignment 4. **Price Differentiation** - Premium positioning - Value positioning - Transparent pricing - Flexible packaging ### Positioning Statement Framework ``` For [target customer] Who [statement of need or opportunity] Our product is [product category] That [statement of key benefit] Unlike [primary competitive alternative] Our product [statement of primary differentiation] ``` **Example:** ``` For e-commerce companies Who struggle with email marketing automation Our product is an AI-powered email platform That increases conversion rates by 40% Unlike Klaviyo and Mailchimp Our product uses AI to personalize at scale ``` ## Competitive Intelligence ### Information Gathering **Public Sources:** - Company websites and blogs - Press releases and news - Job postings (hint at strategy) - Customer reviews (G2, Capterra) - Social media and forums - Glassdoor (employee insights) - SEC filings (public companies) - Patent filings **Direct Research:** - Customer interviews - Win/loss analysis - Sales team feedback - Product demos and trials - Conference attendance ### Competitor Profile Template For each key competitor, document: **Company Overview:** - Founded, HQ, funding, size - Leadership team - Company stage and trajectory **Product:** - Core features - Target customers - Pricing and packaging - Technology stack - Recent launches **Go-to-Market:** - Sales model (self-serve, sales-led) - Marketing strategy - Distribution channels - Partnerships **Strengths:** - What they do better than anyone - Key competitive advantages - Market position **Weaknesses:** - Gaps in product - Customer complaints - Operational challenges **Strategy:** - Stated direction - Inferred priorities - Likely next moves ## Competitive Pricing Analysis ### Price Positioning **Premium (Top 25%):** - Superior product/service - Strong brand - High-touch sales - Enterprise focus **Mid-Market (Middle 50%):** - Balanced value - Standard features - Mixed sales model - Broad market **Value (Bottom 25%):** - Basic functionality - Self-service - Cost leadership - High volume, low margin ### Pricing Comparison Matrix | Competitor | Entry Price | Mid Tier | Enterprise | Model | | ------------ | ----------- | -------- | ---------- | ------------ | | Competitor A | $29/mo | $99/mo | Custom | Subscription | | Competitor B | $49/mo | $199/mo | $499/mo | Subscription | | Us | $39/mo | $129/mo | Custom | Subscription | **Analysis:** - Are we priced competitively? - What does our pricing signal? - Are there gaps in our packaging? ## Go-to-Market Strategy ### Market Entry Strategies **Direct Competition:** - Head-to-head against established players - Requires differentiation and resources - Example: Better features at lower price **Niche Focus:** - Target underserved segment - Become specialist vs. generalist - Example: "Salesforce for real estate" **Disruptive Innovation:** - Target non-consumers or low end - Improve over time to move upmarket - Example: Freemium model disrupting enterprise **Platform Play:** - Build ecosystem and network effects - Aggregate complementary services - Example: Marketplace or API platform ### Beachhead Market **Characteristics of Good Beachhead:** - Specific, reachable segment - Acute pain you solve well - Limited competition - Willing to pay - Can lead to expansion **Example:** Instead of "project management software", target "project management for construction teams" ## Competitive Advantage ### Sustainable Advantages **Network Effects:** - Value increases with users - Example: Slack, marketplaces **Switching Costs:** - High cost to change - Example: CRM systems with data **Economies of Scale:** - Unit costs decrease with volume - Example: Cloud infrastructure **Brand:** - Trust and reputation - Example: Security software **Proprietary Technology:** - Patents or trade secrets - Example: Algorithms, data **Regulatory:** - Licenses or approvals - Example: Fintech, healthcare ### Testing Your Advantage Ask: - Can competitors copy this in < 2 years? - Does this matter to customers? - Do we execute this better than anyone? - Is this advantage durable? If "no" to any, it's not a sustainable advantage. ## Competitive Monitoring ### What to Track **Product Changes:** - New features - Pricing changes - Packaging adjustments **Market Signals:** - Funding announcements - Key hires (especially leadership) - Customer wins/losses - Partnerships **Performance Metrics:** - Revenue (if public or disclosed) - Customer count - Growth rate - Market share estimates ### Monitoring Cadence **Weekly:** - Product release notes - News mentions **Monthly:** - Win/loss analysis review - Positioning map updates **Quarterly:** - Deep competitive review - Strategy adjustment **Annually:** - Major strategy reassessment - Market trends analysis ## Additional Resources ### Reference Files - **`references/frameworks-deep-dive.md`** - Detailed application of each framework with worksheets - **`references/intel-sources.md`** - Comprehensive list of competitive intelligence sources ### Example Files - **`examples/competitor-analysis.md`** - Complete competitive analysis for a SaaS startup - **`examples/positioning-workshop.md`** - Step-by-step positioning development process ## Quick Start To analyze competitive landscape: 1. **Identify competitors** - Direct, indirect, and future threats 2. **Apply Porter's Five Forces** - Assess industry attractiveness 3. **Create positioning map** - Visualize competitive space 4. **Profile top 3-5 competitors** - Deep dive on key rivals 5. **Identify differentiation** - What makes you unique 6. **Analyze pricing** - Where do you fit? 7. **Assess advantages** - What's defensible? 8. **Develop strategy** - How to win For detailed frameworks and examples, see `references/` and `examples/`.
šŸ‘0
šŸ‘ļø0
šŸ¤– Auto-discovered
šŸ¤–system prompt•7 months ago

code-review-excellence

Master effective code review practices to provide constructive

coding
⭐1
# Code Review Excellence Transform code reviews from gatekeeping to knowledge sharing through constructive feedback, systematic analysis, and collaborative improvement. ## When to Use This Skill - Reviewing pull requests and code changes - Establishing code review standards for teams - Mentoring junior developers through reviews - Conducting architecture reviews - Creating review checklists and guidelines - Improving team collaboration - Reducing code review cycle time - Maintaining code quality standards ## Core Principles ### 1. The Review Mindset **Goals of Code Review:** - Catch bugs and edge cases - Ensure code maintainability - Share knowledge across team - Enforce coding standards - Improve design and architecture - Build team culture **Not the Goals:** - Show off knowledge - Nitpick formatting (use linters) - Block progress unnecessarily - Rewrite to your preference ### 2. Effective Feedback **Good Feedback is:** - Specific and actionable - Educational, not judgmental - Focused on the code, not the person - Balanced (praise good work too) - Prioritized (critical vs nice-to-have) ```markdown āŒ Bad: "This is wrong." āœ… Good: "This could cause a race condition when multiple users access simultaneously. Consider using a mutex here." āŒ Bad: "Why didn't you use X pattern?" āœ… Good: "Have you considered the Repository pattern? It would make this easier to test. Here's an example: [link]" āŒ Bad: "Rename this variable." āœ… Good: "[nit] Consider `userCount` instead of `uc` for clarity. Not blocking if you prefer to keep it." ``` ### 3. Review Scope **What to Review:** - Logic correctness and edge cases - Security vulnerabilities - Performance implications - Test coverage and quality - Error handling - Documentation and comments - API design and naming - Architectural fit **What Not to Review Manually:** - Code formatting (use Prettier, Black, etc.) - Import organization - Linting violations - Simple typos ## Review Process ### Phase 1: Context Gathering (2-3 minutes) ```markdown Before diving into code, understand: 1. Read PR description and linked issue 2. Check PR size (>400 lines? Ask to split) 3. Review CI/CD status (tests passing?) 4. Understand the business requirement 5. Note any relevant architectural decisions ``` ### Phase 2: High-Level Review (5-10 minutes) ```markdown 1. **Architecture & Design** - Does the solution fit the problem? - Are there simpler approaches? - Is it consistent with existing patterns? - Will it scale? 2. **File Organization** - Are new files in the right places? - Is code grouped logically? - Are there duplicate files? 3. **Testing Strategy** - Are there tests? - Do tests cover edge cases? - Are tests readable? ``` ### Phase 3: Line-by-Line Review (10-20 minutes) ```markdown For each file: 1. **Logic & Correctness** - Edge cases handled? - Off-by-one errors? - Null/undefined checks? - Race conditions? 2. **Security** - Input validation? - SQL injection risks? - XSS vulnerabilities? - Sensitive data exposure? 3. **Performance** - N+1 queries? - Unnecessary loops? - Memory leaks? - Blocking operations? 4. **Maintainability** - Clear variable names? - Functions doing one thing? - Complex code commented? - Magic numbers extracted? ``` ### Phase 4: Summary & Decision (2-3 minutes) ```markdown 1. Summarize key concerns 2. Highlight what you liked 3. Make clear decision: - āœ… Approve - šŸ’¬ Comment (minor suggestions) - šŸ”„ Request Changes (must address) 4. Offer to pair if complex ``` ## Review Techniques ### Technique 1: The Checklist Method ```markdown ## Security Checklist - [ ] User input validated and sanitized - [ ] SQL queries use parameterization - [ ] Authentication/authorization checked - [ ] Secrets not hardcoded - [ ] Error messages don't leak info ## Performance Checklist - [ ] No N+1 queries - [ ] Database queries indexed - [ ] Large lists paginated - [ ] Expensive operations cached - [ ] No blocking I/O in hot paths ## Testing Checklist - [ ] Happy path tested - [ ] Edge cases covered - [ ] Error cases tested - [ ] Test names are descriptive - [ ] Tests are deterministic ``` ### Technique 2: The Question Approach Instead of stating problems, ask questions to encourage thinking: ```markdown āŒ "This will fail if the list is empty." āœ… "What happens if `items` is an empty array?" āŒ "You need error handling here." āœ… "How should this behave if the API call fails?" āŒ "This is inefficient." āœ… "I see this loops through all users. Have we considered the performance impact with 100k users?" ``` ### Technique 3: Suggest, Don't Command ````markdown ## Use Collaborative Language āŒ "You must change this to use async/await" āœ… "Suggestion: async/await might make this more readable: `typescript async function fetchUser(id: string) { const user = await db.query('SELECT * FROM users WHERE id = ?', id); return user; } ` What do you think?" āŒ "Extract this into a function" āœ… "This logic appears in 3 places. Would it make sense to extract it into a shared utility function?" ```` ### Technique 4: Differentiate Severity ```markdown Use labels to indicate priority: šŸ”“ [blocking] - Must fix before merge 🟔 [important] - Should fix, discuss if disagree 🟢 [nit] - Nice to have, not blocking šŸ’” [suggestion] - Alternative approach to consider šŸ“š [learning] - Educational comment, no action needed šŸŽ‰ [praise] - Good work, keep it up! Example: "šŸ”“ [blocking] This SQL query is vulnerable to injection. Please use parameterized queries." "🟢 [nit] Consider renaming `data` to `userData` for clarity." "šŸŽ‰ [praise] Excellent test coverage! This will catch edge cases." ``` ## Language-Specific Patterns ### Python Code Review ```python # Check for Python-specific issues # āŒ Mutable default arguments def add_item(item, items=[]): # Bug! Shared across calls items.append(item) return items # āœ… Use None as default def add_item(item, items=None): if items is None: items = [] items.append(item) return items # āŒ Catching too broad try: result = risky_operation() except: # Catches everything, even KeyboardInterrupt! pass # āœ… Catch specific exceptions try: result = risky_operation() except ValueError as e: logger.error(f"Invalid value: {e}") raise # āŒ Using mutable class attributes class User: permissions = [] # Shared across all instances! # āœ… Initialize in __init__ class User: def __init__(self): self.permissions = [] ``` ### TypeScript/JavaScript Code Review ```typescript // Check for TypeScript-specific issues // āŒ Using any defeats type safety function processData(data: any) { // Avoid any return data.value; } // āœ… Use proper types interface DataPayload { value: string; } function processData(data: DataPayload) { return data.value; } // āŒ Not handling async errors async function fetchUser(id: string) { const response = await fetch(`/api/users/${id}`); return response.json(); // What if network fails? } // āœ… Handle errors properly async function fetchUser(id: string): Promise<User> { try { const response = await fetch(`/api/users/${id}`); if (!response.ok) { throw new Error(`HTTP ${response.status}`); } return await response.json(); } catch (error) { console.error('Failed to fetch user:', error); throw error; } } // āŒ Mutation of props function UserProfile({ user }: Props) { user.lastViewed = new Date(); // Mutating prop! return <div>{user.name}</div>; } // āœ… Don't mutate props function UserProfile({ user, onView }: Props) { useEffect(() => { onView(user.id); // Notify parent to update }, [user.id]); return <div>{user.name}</div>; } ``` ## Advanced Review Patterns ### Pattern 1: Architectural Review ```markdown When reviewing significant changes: 1. **Design Document First** - For large features, request design doc before code - Review design with team before implementation - Agree on approach to avoid rework 2. **Review in Stages** - First PR: Core abstractions and interfaces - Second PR: Implementation - Third PR: Integration and tests - Easier to review, faster to iterate 3. **Consider Alternatives** - "Have we considered using [pattern/library]?" - "What's the tradeoff vs. the simpler approach?" - "How will this evolve as requirements change?" ``` ### Pattern 2: Test Quality Review ```typescript // āŒ Poor test: Implementation detail testing test('increments counter variable', () => { const component = render(<Counter />); const button = component.getByRole('button'); fireEvent.click(button); expect(component.state.counter).toBe(1); // Testing internal state }); // āœ… Good test: Behavior testing test('displays incremented count when clicked', () => { render(<Counter />); const button = screen.getByRole('button', { name: /increment/i }); fireEvent.click(button); expect(screen.getByText('Count: 1')).toBeInTheDocument(); }); // Review questions for tests: // - Do tests describe behavior, not implementation? // - Are test names clear and descriptive? // - Do tests cover edge cases? // - Are tests independent (no shared state)? // - Can tests run in any order? ``` ### Pattern 3: Security Review ```markdown ## Security Review Checklist ### Authentication & Authorization - [ ] Is authentication required where needed? - [ ] Are authorization checks before every action? - [ ] Is JWT validation proper (signature, expiry)? - [ ] Are API keys/secrets properly secured? ### Input Validation - [ ] All user inputs validated? - [ ] File uploads restricted (size, type)? - [ ] SQL queries parameterized? - [ ] XSS protection (escape output)? ### Data Protection - [ ] Passwords hashed (bcrypt/argon2)? - [ ] Sensitive data encrypted at rest? - [ ] HTTPS enforced for sensitive data? - [ ] PII handled according to regulations? ### Common Vulnerabilities - [ ] No eval() or similar dynamic execution? - [ ] No hardcoded secrets? - [ ] CSRF protection for state-changing operations? - [ ] Rate limiting on public endpoints? ``` ## Giving Difficult Feedback ### Pattern: The Sandwich Method (Modified) ```markdown Traditional: Praise + Criticism + Praise (feels fake) Better: Context + Specific Issue + Helpful Solution Example: "I noticed the payment processing logic is inline in the controller. This makes it harder to test and reuse. [Specific Issue] The calculateTotal() function mixes tax calculation, discount logic, and database queries, making it difficult to unit test and reason about. [Helpful Solution] Could we extract this into a PaymentService class? That would make it testable and reusable. I can pair with you on this if helpful." ``` ### Handling Disagreements ```markdown When author disagrees with your feedback: 1. **Seek to Understand** "Help me understand your approach. What led you to choose this pattern?" 2. **Acknowledge Valid Points** "That's a good point about X. I hadn't considered that." 3. **Provide Data** "I'm concerned about performance. Can we add a benchmark to validate the approach?" 4. **Escalate if Needed** "Let's get [architect/senior dev] to weigh in on this." 5. **Know When to Let Go** If it's working and not a critical issue, approve it. Perfection is the enemy of progress. ``` ## Best Practices 1. **Review Promptly**: Within 24 hours, ideally same day 2. **Limit PR Size**: 200-400 lines max for effective review 3. **Review in Time Blocks**: 60 minutes max, take breaks 4. **Use Review Tools**: GitHub, GitLab, or dedicated tools 5. **Automate What You Can**: Linters, formatters, security scans 6. **Build Rapport**: Emoji, praise, and empathy matter 7. **Be Available**: Offer to pair on complex issues 8. **Learn from Others**: Review others' review comments ## Common Pitfalls - **Perfectionism**: Blocking PRs for minor style preferences - **Scope Creep**: "While you're at it, can you also..." - **Inconsistency**: Different standards for different people - **Delayed Reviews**: Letting PRs sit for days - **Ghosting**: Requesting changes then disappearing - **Rubber Stamping**: Approving without actually reviewing - **Bike Shedding**: Debating trivial details extensively ## Templates ### PR Review Comment Template ```markdown ## Summary [Brief overview of what was reviewed] ## Strengths - [What was done well] - [Good patterns or approaches] ## Required Changes šŸ”“ [Blocking issue 1] šŸ”“ [Blocking issue 2] ## Suggestions šŸ’” [Improvement 1] šŸ’” [Improvement 2] ## Questions ā“ [Clarification needed on X] ā“ [Alternative approach consideration] ## Verdict āœ… Approve after addressing required changes ``` ## Resources - **references/code-review-best-practices.md**: Comprehensive review guidelines - **references/common-bugs-checklist.md**: Language-specific bugs to watch for - **references/security-review-guide.md**: Security-focused review checklist - **assets/pr-review-template.md**: Standard review comment template - **assets/review-checklist.md**: Quick reference checklist - **scripts/pr-analyzer.py**: Analyze PR complexity and suggest reviewers
šŸ‘0
šŸ‘ļø3
šŸ¤– Auto-discovered