feat: Reimplement & test GetCandidateRoute algorithm

.vscode/launch.json

@@ -113,6 +113,20 @@
         "-test.v"
       ]
     },
+    {
+        "name": "tokens/usecase/pricing/chain",
+        "type": "go",
+        "request": "launch",
+        "mode": "test",
+        "program": "${workspaceFolder}/tokens/usecase/pricing/chain",
+        "args": [
+          "-test.timeout",
+          "30m",
+          "-test.run",
+          "TestPricingTestSuite/TestComputePrice_Specific",
+          "-test.v"
+        ]
+      },
     {
       "name": "router/usecase/pools",
       "type": "go",
@@ -135,7 +149,6 @@
       "args": [
         "-v",
         "tests/test_tokens_prices.py"
-        // "tests/test_pools.py"
       ],
       "env": {
         "SQS_ENVIRONMENTS": "local",

Makefile

@@ -165,6 +165,7 @@ sqs-update-mainnet-state:
 
 	curl -X POST "http:/localhost:9092/tokens/store-state"
 	mv tokens.json router/usecase/routertesting/parsing/tokens.json
+	mv pool_denom_metadata.json router/usecase/routertesting/parsing/pool_denom_metadata.json
 
 # Bench tests pricing
 bench-pricing:

app/sidecar_query_server.go

@@ -101,8 +101,11 @@ func NewSideCarQueryServer(appCodec codec.Codec, config domain.Config, logger lo
 	// Initialize pools repository, usecase and HTTP handler
 	poolsUseCase := poolsUseCase.NewPoolsUsecase(config.Pools, config.ChainGRPCGatewayEndpoint, routerRepository, tokensUseCase.GetChainScalingFactorByDenomMut)
 
+	// Initialize candidate route searcher
+	candidateRouteSearcher := routerUseCase.NewCandidateRouteFinder(routerRepository, logger)
+
 	// Initialize router repository, usecase
-	routerUsecase := routerUseCase.NewRouterUsecase(routerRepository, poolsUseCase, *config.Router, poolsUseCase.GetCosmWasmPoolConfig(), logger, cache.New(), cache.New())
+	routerUsecase := routerUseCase.NewRouterUsecase(routerRepository, poolsUseCase, candidateRouteSearcher, tokensUseCase, *config.Router, poolsUseCase.GetCosmWasmPoolConfig(), logger, cache.New(), cache.New())
 
 	// Initialize system handler
 	chainInfoRepository := chaininforepo.New()
@@ -111,7 +114,7 @@ func NewSideCarQueryServer(appCodec codec.Codec, config domain.Config, logger lo
 	cosmWasmPoolConfig := poolsUseCase.GetCosmWasmPoolConfig()
 
 	// Initialize chain pricing strategy
-	pricingSimpleRouterUsecase := routerUseCase.NewRouterUsecase(routerRepository, poolsUseCase, *config.Router, cosmWasmPoolConfig, logger, cache.New(), cache.New())
+	pricingSimpleRouterUsecase := routerUseCase.NewRouterUsecase(routerRepository, poolsUseCase, candidateRouteSearcher, tokensUseCase, *config.Router, cosmWasmPoolConfig, logger, cache.New(), cache.New())
 	chainPricingSource, err := pricing.NewPricingStrategy(*config.Pricing, tokensUseCase, pricingSimpleRouterUsecase)
 	if err != nil {
 		return nil, err

docs/architecture/COSMWASM_POOLS.MD

@@ -54,6 +54,34 @@ One caveat on utilizing cw2 information is that there is no uniqueness check for
 `crates.io:transmuter` `<3.0.0`: requires no additional information.
 `crates.io:transmuter` `>=3.0.0`: requires alloyed asset denom and normalization factors for each asset.
 
+### Alloyed
+
+[Alloyed Transmuter Pools](https://forum.osmosis.zone/t/alloyed-assets-on-osmosis-unifying-ux-and-solving-liquidity-fragmentation/2624) is a novel mechanism developed
+by Osmosis to unify multiple bridged assets into a single LP share token.
+
+Alloyed assets represent LP shares of these pools, combining various asset versions into one fungible token. Depositing different asset versions into the pool adds liquidity and, in return, users receive LP shares in the form of alloyed assets, enhancing UX by consolidating liquidity.
+
+Users can acquire alloyed assets by simply swapping a like-kind asset for the alloyed version in the swap page.
+
+Swapping into the alloy is equivalent to depositing the asset into the pool and receiving the LP share token. Swapping out of the alloy is equivalent to burning the LP share token and receiving the asset.
+
+In the context of SQS router, this poses a number of challenges. The router needs to be aware of the alloyed asset denom.
+
+For example, the alloyed asset denom is not contained in the pool balances or denoms of the pools that mints it as an LP share. Without custom handling, the router would be unable to consider the alloyed asset denom in route finding
+This stems from the fact that the route finding algorithm looks at the pool denoms as well as how much liquidity a denom contributes to.
+
+To address this, we define the following invariants for alloyed assets for the context of SQS:
+- Each pool that mints alloyed asset denom does not contain it in balances but we include it in the pool denoms.
+- The pool that mints alloyed asset denom does not include the alloyed asset value in its liquidity capitalization.
+   * Example: [Pool 1868](https://app.osmosis.zone/pool/1868) and allBTC.
+- The pool that does not mint alloyed asset but contains it one of the direct assets includes the alloyed asset value in its liquidity capitalization as well as in the balances
+   * Example: [Pool 1835](https://app.osmosis.zone/pool/1835) and USDT.
+- During ingest, we maintain the [BlockPoolMetaData.DenomPoolLiquidityMap](https://github.com/osmosis-labs/sqs/blob/d32f6a1ef6fd2a081f60f1f510023a0c8f9b1530/docs/architecture/ingest.md#L37-L49) to keep track of the liquidity capitalization for each denom across all pools.
+  * We add the alloyed asset value to the liquidity capitalization contribution for non-minting pools that contain an LP share as one of the direct assets.
+  * However, for `BlockPoolMetaData.DenomPoolLiquidityMap.Pools`, we add the LP share as contributing
+  zero to liqudiity capitalization. By maintaining the LP share denom in the `DenomPoolLiquidityMap.Pools`, we can ensure that the LP share is not double counted towards liquidity capitalization but still be able
+  to find routes over the "minting" pools.
+
 ## Orderbook
 `crates.io:sumtree-orderbook` `>= 0.1.0`: requires base and quote denoms, ticks liquidity, next bid and ask tick.
 

docs/architecture/ingest.md

@@ -69,6 +69,12 @@ asset and the default quote denom.
 
 Once complete, it calls a hook to notify the subscribed listeners that the prices have been updated.
 
+Note, that we recompute pricing logic twice for the first block ingested. First time it is done
+syncronously to avoid proceeding before prices are computed. At that point the pool liquidity pricing
+is not enabled yet since we have no pricing data. Therefore, the prices are compouted via suboptimal routes.
+As a result, once the prices for all tokens are computed the first time, we trigger the pricing worker
+asyncronously for all tokens second time.
+
 #### Pricing Listeners
 
 - Healthcheck: The healthcheck listener is responsible for updating the healthcheck status based on the last time the prices were updated. If the prices are not updated within a certain time period, the healthcheck status will be updated to unhealthy.
@@ -134,6 +140,8 @@ For example, assume that there is an ATOM/OSMO pool that is modified within a bl
 The denom liquidity capitalization and pool liquidity capitalizaion for each pool are computed concurrently by the
 pool liquidity pricer worker after every block.
 
+For the alloyed pools, there is custom handling to account for the alloyed asset denom. See `docs/architecture/COSMWASM_POOLS.MD` for details.
+
 ### Candidate Route Search Data
 
 This worker is responsible for pre-computing the candidate route search pool data for each denom.

domain/candidate_routes.go

@@ -0,0 +1,24 @@
+package domain
+
+import (
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/osmosis-labs/sqs/sqsdomain"
+)
+
+// CandidateRouteSearchOptions represents the options for finding candidate routes.
+type CandidateRouteSearchOptions struct {
+	// MaxRoutes is the maximum number of routes to find.
+	MaxRoutes int
+	// MaxPoolsPerRoute is the maximum number of pools to consider for each route.
+	MaxPoolsPerRoute int
+	// MinPoolLiquidityCap is the minimum liquidity cap for a pool to be considered.
+	MinPoolLiquidityCap uint64
+}
+
+// CandidateRouteSearcher is the interface for finding candidate routes.
+type CandidateRouteSearcher interface {
+	// FindCandidateRoutes finds candidate routes for a given tokenIn and tokenOutDenom
+	// using the given options.
+	// Returns the candidate routes and an error if any.
+	FindCandidateRoutes(tokenIn sdk.Coin, tokenOutDenom string, options CandidateRouteSearchOptions) (sqsdomain.CandidateRoutes, error)
+}

domain/mocks/candidate_route_data_holder_mock.go

@@ -20,3 +20,12 @@ func (c *CandidateRouteSearchDataHolderMock) GetCandidateRouteSearchData() map[s
 func (c *CandidateRouteSearchDataHolderMock) SetCandidateRouteSearchData(candidateRouteSearchData map[string][]sqsdomain.PoolI) {
 	c.CandidateRouteSearchData = candidateRouteSearchData
 }
+
+// GetRankedPoolsByDenom implements mvc.CandidateRouteSearchDataHolder.
+func (c *CandidateRouteSearchDataHolderMock) GetRankedPoolsByDenom(denom string) ([]sqsdomain.PoolI, error) {
+	pools, ok := c.CandidateRouteSearchData[denom]
+	if !ok {
+		return []sqsdomain.PoolI{}, nil
+	}
+	return pools, nil
+}

domain/mocks/candidate_route_finder_mock.go

@@ -0,0 +1,19 @@
+package mocks
+
+import (
+	"github.com/cosmos/cosmos-sdk/types"
+	"github.com/osmosis-labs/sqs/domain"
+	"github.com/osmosis-labs/sqs/sqsdomain"
+)
+
+type CandidateRouteFinderMock struct {
+	Routes sqsdomain.CandidateRoutes
+	Error  error
+}
+
+var _ domain.CandidateRouteSearcher = CandidateRouteFinderMock{}
+
+// FindCandidateRoutes implements domain.CandidateRouteSearcher.
+func (c CandidateRouteFinderMock) FindCandidateRoutes(tokenIn types.Coin, tokenOutDenom string, options domain.CandidateRouteSearchOptions) (sqsdomain.CandidateRoutes, error) {
+	return c.Routes, c.Error
+}

domain/mocks/token_metadata_holder_mock.go

@@ -0,0 +1,15 @@
+package mocks
+
+import "github.com/osmosis-labs/sqs/domain/mvc"
+
+type TokenMetadataHolderMock struct {
+	MockMinPoolLiquidityCap      uint64
+	MockMinPoolLiquidityCapError error
+}
+
+var _ mvc.TokenMetadataHolder = &TokenMetadataHolderMock{}
+
+// GetMinPoolLiquidityCap implements mvc.TokenMetadataHolder.
+func (t *TokenMetadataHolderMock) GetMinPoolLiquidityCap(denomA string, denomB string) (uint64, error) {
+	return t.MockMinPoolLiquidityCap, t.MockMinPoolLiquidityCapError
+}

domain/mvc/router.go

@@ -17,6 +17,11 @@ type CandidateRouteSearchDataHolder interface {
 
 	// GetCandidateRouteSearchData gets the candidate route search data from the holder
 	GetCandidateRouteSearchData() map[string][]sqsdomain.PoolI
+
+	// GetRankedPoolsByDenom returns the ranked candidate route search pools for a given denom.
+	// Returns an empty slice if the denom is not found.
+	// Returns error if retrieved pools are not of type sqsdomain.PoolI.
+	GetRankedPoolsByDenom(denom string) ([]sqsdomain.PoolI, error)
 }
 
 // RouterRepository represents the contract for a repository handling tokens information
@@ -82,6 +87,12 @@ type RouterUsecase interface {
 
 	GetConfig() domain.RouterConfig
 
+	// GetMinPoolLiquidityCapFilter returns the min pool liquidity capitalization filter for the given tokenIn and tokenOutDenom.
+	// It is used to filter out pools with liquidity less than the output of this function.
+	// Returns error if one of the denom metadata is not found.
+	// Returns error if the filter is not found for the given denoms.
+	GetMinPoolLiquidityCapFilter(tokenInDenom, tokenOutDenom string) (uint64, error)
+
 	// ConvertMinTokensPoolLiquidityCapToFilter converts the minTokensPoolLiquidityCap to a filter.
 	// It is used to filter out pools with liquidity less than the output of this function.
 	// We use min(tokenInPoolLiquidityCap, tokenOutPoolLiquidityCap) as a proxy for finding the appropriate

domain/mvc/tokens.go

@@ -17,8 +17,16 @@ type TokensPoolLiquidityHandler interface {
 	UpdatePoolDenomMetadata(tokensMetadata domain.PoolDenomMetaDataMap)
 }
 
+type TokenMetadataHolder interface {
+	// GetMinPoolLiquidityCap returns the min pool liquidity capitalization between the two denoms.
+	// Returns error if there is no pool liquidity metadata for one of the tokens.
+	// Returns error if pool liquidity metadata is large enough to cause overflow.
+	GetMinPoolLiquidityCap(denomA, denomB string) (uint64, error)
+}
+
 // TokensUsecase defines an interface for the tokens usecase.
 type TokensUsecase interface {
+	TokenMetadataHolder
 	TokensPoolLiquidityHandler
 
 	// GetMetadataByChainDenom returns token metadata for a given chain denom.
@@ -48,11 +56,6 @@ type TokensUsecase interface {
 	// The result of the inner map is prices of the outer base and inner quote.
 	GetPrices(ctx context.Context, baseDenoms []string, quoteDenoms []string, pricingSourceType domain.PricingSourceType, opts ...domain.PricingOption) (domain.PricesResult, error)
 
-	// GetMinPoolLiquidityCap returns the min pool liquidity capitalization between the two denoms.
-	// Returns error if there is no pool liquidity metadata for one of the tokens.
-	// Returns error if pool liquidity metadata is large enough to cause overflow.
-	GetMinPoolLiquidityCap(denomA, denomB string) (uint64, error)
-
 	// GetPoolDenomMetadata returns the pool denom metadata of a pool denom.
 	// This metadata is accumulated from all pools.
 	GetPoolDenomMetadata(chainDenom string) (domain.PoolDenomMetaData, error)
@@ -80,6 +83,10 @@ type TokensUsecase interface {
 
 	// GetCoingeckoIdByChainDenom gets the Coingecko ID by chain denom
 	GetCoingeckoIdByChainDenom(chainDenom string) (string, error)
+
+	// ClearPoolDenomMetadata implements mvc.TokensUsecase.
+	// WARNING: use with caution, this will clear all pool denom metadata
+	ClearPoolDenomMetadata()
 }
 
 // ValidateChainDenomQueryParam validates the chain denom query parameter.

ingest/usecase/export_test.go

@@ -21,3 +21,7 @@ func TransferDenomLiquidityMap(transferTo, transferFrom domain.DenomPoolLiquidit
 func ProcessSQSModelMut(sqsModel *sqsdomain.SQSPool) error {
 	return processSQSModelMut(sqsModel)
 }
+
+func UpdateCurrentBlockLiquidityMapAlloyed(currentBlockLiquidityMap domain.DenomPoolLiquidityMap, poolID uint64, alloyedDenom string) domain.DenomPoolLiquidityMap {
+	return updateCurrentBlockLiquidityMapAlloyed(currentBlockLiquidityMap, poolID, alloyedDenom)
+}

ingest/usecase/ingest_usecase.go

@@ -147,8 +147,14 @@ func (p *ingestUseCase) ProcessBlockData(ctx context.Context, height uint64, tak
 		// to avoid overloading the system.
 		defer p.firstBlockWg.Done()
 
-		// Pre-compute the prices for all
+		// Pre-compute the prices for all tokens
 		p.defaultQuotePriceUpdateWorker.UpdatePricesSync(height, uniqueBlockPoolMetadata)
+
+		// Completely reprice the pool liquidity for the first block asyncronously
+		// second time.
+		// This is necessary because the initial pricing is computed within min liquidity capitalization.
+		// That results in a suboptimal price.
+		p.defaultQuotePriceUpdateWorker.UpdatePricesAsync(height, uniqueBlockPoolMetadata)
 	} else {
 		// Wait for the first block to be processed before
 		// updating the prices for the next block.
@@ -219,7 +225,8 @@ func (p *ingestUseCase) parsePoolData(ctx context.Context, poolData []*types.Poo
 			}
 
 			// Get balances and pool ID.
-			currentPoolBalances := poolResult.pool.GetSQSPoolModel().Balances
+			sqsModel := poolResult.pool.GetSQSPoolModel()
+			currentPoolBalances := sqsModel.Balances
 			poolID := poolResult.pool.GetId()
 
 			// Update block liquidity map.
@@ -235,6 +242,16 @@ func (p *ingestUseCase) parsePoolData(ctx context.Context, poolData []*types.Poo
 				uniqueData.UpdatedDenoms[balance.Denom] = struct{}{}
 			}
 
+			// Handle the alloyed LP share stemming from the "minting" pools.
+			// See updateCurrentBlockLiquidityMapAlloyed for details.
+			cosmWasmModel := sqsModel.CosmWasmPoolModel
+			if cosmWasmModel != nil && cosmWasmModel.IsAlloyTransmuter() {
+				alloyedDenom := cosmWasmModel.Data.AlloyTransmuter.AlloyedDenom
+				uniqueData.UpdatedDenoms[alloyedDenom] = struct{}{}
+
+				currentBlockLiquidityMap = updateCurrentBlockLiquidityMapAlloyed(currentBlockLiquidityMap, poolID, alloyedDenom)
+			}
+
 			// Update unique pools.
 			uniqueData.PoolIDs[poolID] = struct{}{}
 
@@ -293,6 +310,38 @@ func updateCurrentBlockLiquidityMapFromBalances(currentBlockLiquidityMap domain.
 	return currentBlockLiquidityMap
 }
 
+// updateCurrentBlockLiquidityMapAlloyed updates the current block liquidity map with the alloyed LP share.
+// Since the LP share is not present in the balances for the "minting" pool, we treat its contribution to
+// the liqudity as zero. However, we still create a mapping from the LP share to the pool ID so that
+// we can find routes over the minting pools.
+// CONTRACT: the pool ID is validated to be an alloyed pool that mints the LP share.
+// See `docs/architecture/COSMWASM_POOLS.md` for details.
+func updateCurrentBlockLiquidityMapAlloyed(currentBlockLiquidityMap domain.DenomPoolLiquidityMap, poolID uint64, alloyedDenom string) domain.DenomPoolLiquidityMap {
+	denomPoolLiquidityData, ok := currentBlockLiquidityMap[alloyedDenom]
+
+	// Note: we do not update the total liquidity since this is
+	// a contribution of the "minting" LP share pool.
+	// We only update the total liquidity for an alloyed
+	// whenever it is included in the balances as part of the "non-minting" pools.
+	// However, for the purposes of finding candidate routes, we treate as if the LP
+	// share is contributing zero to the liquidity of the pool it is minted from.
+	// See `docs/architecture/COSMWASM_POOLS.md` for details.
+	if ok {
+		denomPoolLiquidityData.Pools[poolID] = osmomath.ZeroInt()
+		currentBlockLiquidityMap[alloyedDenom] = denomPoolLiquidityData
+	} else {
+		currentBlockLiquidityMap[alloyedDenom] = domain.DenomPoolLiquidityData{
+			// Note: we do not update the total liquidity since it is not present in the balances.
+			TotalLiquidity: osmomath.ZeroInt(),
+			Pools: map[uint64]osmomath.Int{
+				poolID: osmomath.ZeroInt(),
+			},
+		}
+	}
+
+	return currentBlockLiquidityMap
+}
+
 // transferDenomLiquidityMap transfer the updated block denom liquidity data from transferFrom to
 // transferTo map.
 //