Merge pull request #1445 from 0chain/sprint-1.16

Sprint 1.16

Author: dabasov

.github/workflows/build-&-publish-docker-image.yml

@@ -215,83 +215,18 @@ jobs:
             version: '1.7'
             force: 'false'
 
-      - name: "Create Tenderly fork"
+      - name: "Create Tenderly virtual testnet"
         run: |
           echo "TENDERLY_CREATION_INFO=$(curl -X POST \
-              -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
-              -H "Content-Type: application/json" \
-              -d '{"alias":"mainnet-dev-${{ env.RUNNER_NUMBER }}-${{ github.run_id }}${{ github.run_attempt }}", "description":"", "block_number": 18539779, "network_id":"1"}' \
-              https://api.tenderly.co/api/v1/account/zus_network/project/project/fork)" >> $GITHUB_ENV
+           -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
+           -H "Content-Type: application/json" \
+           -d '{"slug":"mainnet-dev-${{ env.RUNNER_NUMBER }}-${{ github.run_id }}","displayName":"mainnet-dev-${{ env.RUNNER_NUMBER }}-${{ github.run_id }}","description":"","visibility":"TEAM","tags":{"purpose":"development"},"networkConfig":{"networkId":"1","blockNumber":"18512782","chainConfig":{"chainId":"1"},"baseFeePerGas":"1"},"explorerConfig":{"enabled":false,"verificationVisibility":"bytecode"},"syncState":false}' \
+           https://api.tenderly.co/api/v1/account/zus_network/project/project/testnet/container)" >> $GITHUB_ENV
 
-      - name: "Parse Tenderly fork creation transaction result"
+      - name: "Parse Tenderly virtual testnet creation transaction result"
         run: |
-          echo "TENDERLY_FORK_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.simulation_fork.id')" >> $GITHUB_ENV
-          echo "TENDERLY_ROOT_TRANSACTION_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.root_transaction.id')" >> $GITHUB_ENV
-
-      - name: "Retrieve Tenderly fork block number"
-        run: |
-          echo "TENDERLY_FORK_BLOCK_NUMBER=$(curl -X GET \
-              -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
-              -H "Content-Type: application/json" \
-              https://api.tenderly.co/api/v1/network/1/block-number | jq -r '.block_number')" >> $GITHUB_ENV
-
-          echo "TENDERLY_FORK_BLOCK_NUMBER=$((${{ env.TENDERLY_FORK_BLOCK_NUMBER }} + 1))" >> GITHUB_ENV
-
-      - name: "Transfer Bridge ownership in Tenderly fork"
-        run: |
-          echo "TENDERLY_ROOT_TRANSACTION_ID=$(curl -X POST \
-              -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
-              -H "Content-Type: application/json" \
-              -d '{
-                "network_id": "1",
-                "block_number": ${{ env.TENDERLY_FORK_BLOCK_NUMBER }},
-                "transaction_index": null,
-                "from": "0xed8f3170db6d1a71c8fa6d8d73cc2c51db95d5a4",
-                "input": "0xf2fde38b0000000000000000000000008e25cfd9bd6c0ca67a5522cd920b3c66d39d6e97",
-                "to": "0x7700d773022b19622095118fadf46f7b9448be9b",
-                "gas": 8000000,
-                "gas_price": "0",
-                "value": "0",
-                "access_list": [],
-                "generate_access_list": true,
-                "save": true,
-                "source": "dashboard",
-                "block_header": null,
-                "root": "${{ env.TENDERLY_ROOT_TRANSACTION_ID }}",
-                "skip_fork_head_update": false,
-                "alias": "",
-                "description": "Transfer ownership to 0x8E25cfd9bd6c0ca67a5522cd920b3c66D39d6E97"
-            }' \
-              https://api.tenderly.co/api/v1/account/zus_network/project/project/fork/${{ env.TENDERLY_FORK_ID }}/simulate | jq -r '.simulation.id')" >> $GITHUB_ENV
-
-          echo "TENDERLY_FORK_BLOCK_NUMBER=$((${{ env.TENDERLY_FORK_BLOCK_NUMBER }} + 1))" >> GITHUB_ENV
-
-      - name: "Transfer Authorizers ownership in Tenderly fork"
-        run: |
-          curl -X POST \
-              -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
-              -H "Content-Type: application/json" \
-              -d '{
-                "network_id": "1",
-                "block_number": ${{ env.TENDERLY_FORK_BLOCK_NUMBER }},
-                "transaction_index": null,
-                "from": "0xed8f3170db6d1a71c8fa6d8d73cc2c51db95d5a4",
-                "input": "0xf2fde38b0000000000000000000000008e25cfd9bd6c0ca67a5522cd920b3c66d39d6e97",
-                "to": "0x481dab4407b9880de0a68dc62e6af611c4949e42",
-                "gas": 8000000,
-                "gas_price": "0",
-                "value": "0",
-                "access_list": [],
-                "generate_access_list": true,
-                "save": true,
-                "source": "dashboard",
-                "block_header": null,
-                "root": "${{ env.TENDERLY_ROOT_TRANSACTION_ID }}",
-                "skip_fork_head_update": false,
-                "alias": "",
-                "description": "Transfer ownership to 0x8E25cfd9bd6c0ca67a5522cd920b3c66D39d6E97"
-            }' \
-              https://api.tenderly.co/api/v1/account/zus_network/project/project/fork/${{ env.TENDERLY_FORK_ID }}/simulate
+          echo "TENDERLY_VIRTUAL_TESTNET_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.container.id')" >> $GITHUB_ENV
+          echo "TENDERLY_VIRTUAL_TESTNET_RPC_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.container.connectivityConfig.endpoints[0].id')" >> $GITHUB_ENV
 
       - name: "Deploy 0Chain"
         uses: 0chain/actions/deploy-0chain@master
@@ -302,10 +237,10 @@ jobs:
           blobber_image: ${{ env.TAG }}-${{ env.SHORT_SHA }}
           validator_image: ${{ env.TAG }}-${{ env.SHORT_SHA }}
           SUBGRAPH_API_URL: ${{ secrets.SUBGRAPH_API_URL }}
-          TENDERLY_FORK_ID: ${{ env.TENDERLY_FORK_ID }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
           graphnode_sc: ${{ secrets.GRAPHNODE_SC }}
           graphnode_network: ${{ secrets.GRAPHNODE_NETWORK }}
-          graphnode_ethereum_node_url: https://rpc.tenderly.co/fork/${{ env.TENDERLY_FORK_ID }}
+          graphnode_ethereum_node_url: https://virtual.mainnet.rpc.tenderly.co/${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
           svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
 
       - name: "Run System tests"
@@ -319,18 +254,18 @@ jobs:
           run_flaky_tests: false
           retry_failures: true
           run_smoke_tests: ${{ github.ref != 'refs/heads/staging' && github.base_ref != 'staging' && github.ref != 'refs/heads/master' && github.base_ref != 'master' }}
-          TENDERLY_FORK_ID: ${{ env.TENDERLY_FORK_ID }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
           DEVOPS_CHANNEL_WEBHOOK_URL: ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }}
           S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
           S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
 
-      - name: "Remove Tenderly fork"
+      - name: "Remove Tenderly virtual testnet"
         if: always()
         run: |
           curl -X DELETE \
             -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
             -H "Content-Type: application/json" \
-            https://api.tenderly.co/api/v1/account/zus_network/project/project/fork/${{ env.TENDERLY_FORK_ID }}  
+            https://api.tenderly.co/api/v1/account/zus_network/project/project/testnet/container/${{ env.TENDERLY_VIRTUAL_TESTNET_ID }}
 
       - name: "Set PR status as ${{ job.status }}"
         if: ${{ (success() || failure()) && steps.findPr.outputs.number }}

.github/workflows/tests.yml

@@ -35,9 +35,6 @@ jobs:
       - name: Build Base
         run: ./docker.local/bin/build.base.sh
 
-      - name: Check swagger generation
-        run: docker.local/bin/test.swagger.sh
-
       - name: Golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:

Makefile

@@ -188,3 +188,10 @@ else
 	sed -i "s/BUF_VERSION := [0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*/BUF_VERSION := $(VERSION)/g" Makefile
 endif
 endif
+
+.PHONY: markdown-docs
+markdown-docs:
+	swagger generate spec -o ./swagger.yaml -w ./code/go/0chain.net -m
+	sed -i '' "s/in\:\ form/in\:\ formData/g" ./swagger.yaml
+	yq -i '(.paths.*.*.parameters.[] | select(.in == "formData") | select(.type == "object")).type = "file"' swagger.yaml
+	swagger generate markdown -f ./swagger.yaml --output=swagger.md

code/go/0chain.net/blobber/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
 	"github.com/0chain/blobber/code/go/0chain.net/core/node"
 )
@@ -43,10 +44,14 @@ func main() {
 	// when enabled "// +build integration_tests", this sets blobber for conductor tests.
 	prepareBlobber(node.Self.ID)
 
-	if err := registerOnChain(); err != nil {
-		logging.Logger.Error("Error register on blockchain" + err.Error())
-		panic(err)
-	}
+	go func() {
+		if err := registerOnChain(); err != nil {
+			logging.Logger.Error("Error register on blockchain" + err.Error())
+			panic(err)
+		}
+
+		common.SetBlobberRegistered(true)
+	}()
 
 	if err := setStorageScConfigFromChain(); err != nil {
 		logging.Logger.Error("Error setStorageScConfigFromChain" + err.Error())

code/go/0chain.net/blobber/zcn.go

@@ -2,17 +2,18 @@ package main
 
 import (
 	"fmt"
-	"time"
-
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler"
 	"github.com/0chain/blobber/code/go/0chain.net/core/chain"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 	handleCommon "github.com/0chain/blobber/code/go/0chain.net/core/common/handler"
+	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
 	"github.com/0chain/blobber/code/go/0chain.net/core/node"
 	"github.com/0chain/gosdk/zboxcore/sdk"
 	"github.com/0chain/gosdk/zcncore"
+	"go.uber.org/zap"
+	"time"
 )
 
 func registerOnChain() error {
@@ -31,8 +32,9 @@ func registerOnChain() error {
 		return err
 	}
 
-	// setup blobber (add or update) on the blockchain (multiple attempts)
-	for i := 1; i <= 10; i++ {
+	// setup blobber (add) on the blockchain (multiple attempts)
+	i := 1
+	for {
 		if i == 1 {
 			fmt.Printf("\r	+ connect to sharders:")
 		} else {
@@ -50,6 +52,9 @@ func registerOnChain() error {
 		err = handler.RegisterBlobber(common.GetRootContext())
 		if err == nil {
 			break
+		} else {
+			i++
+			logging.Logger.Error("add_blobber_error", zap.Error(err))
 		}
 	}
 

code/go/0chain.net/blobbercore/allocation/allocationchange.go

@@ -100,7 +100,7 @@ func (change *AllocationChange) Save(ctx context.Context) error {
 
 func (change *AllocationChange) Update(ctx context.Context) error {
 	db := datastore.GetStore().GetTransaction(ctx)
-	return db.Table(change.TableName()).Where("lookup_hash = ?", change.LookupHash).Updates(map[string]interface{}{
+	return db.Table(change.TableName()).Where("connection_id = ? AND lookup_hash = ?", change.ConnectionID, change.LookupHash).Updates(map[string]interface{}{
 		"size":       change.Size,
 		"updated_at": time.Now(),
 		"input":      change.Input,
@@ -174,10 +174,11 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien
 func GetConnectionObj(ctx context.Context, connectionID, allocationID, clientID string) (*AllocationChangeCollector, error) {
 	cc := &AllocationChangeCollector{}
 	db := datastore.GetStore().GetTransaction(ctx)
-	err := db.Where("id = ? and allocation_id = ? and client_id = ?",
+	err := db.Where("id = ? and allocation_id = ? and client_id = ? AND status <> ?",
 		connectionID,
 		allocationID,
 		clientID,
+		DeletedConnection,
 	).Take(cc).Error
 
 	if err == nil {
@@ -189,7 +190,7 @@ func GetConnectionObj(ctx context.Context, connectionID, allocationID, clientID
 		cc.AllocationID = allocationID
 		cc.ClientID = clientID
 		cc.Status = NewConnection
-		err = cc.Create(ctx)
+		err = cc.Save(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -400,7 +401,7 @@ func deleteFromFileStore(ctx context.Context, allocationID string) error {
 	return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
 		db := datastore.GetStore().GetTransaction(ctx)
 
-		err := db.Model(&reference.Ref{}).Unscoped().Select("id", "validation_root", "thumbnail_hash").
+		err := db.Model(&reference.Ref{}).Unscoped().Select("id", "validation_root", "thumbnail_hash", "filestore_version").
 			Where("allocation_id=? AND is_precommit=? AND type=? AND deleted_at is not NULL", allocationID, true, reference.FILE).
 			FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
 

code/go/0chain.net/blobbercore/allocation/connection.go

@@ -15,7 +15,7 @@ import (
 
 var (
 	// ConnectionObjCleanInterval start to clean the connectionObjMap
-	ConnectionObjCleanInterval = 10 * time.Minute
+	ConnectionObjCleanInterval = 45 * time.Minute
 	// ConnectionObjTimout after which connectionObj entry should be invalid
 	ConnectionObjTimeout = 30 * time.Minute
 )
@@ -182,7 +182,11 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string
 		change.lock.Lock()
 		defer change.lock.Unlock()
 		connectionObj.lock.Unlock()
-		err := cmd.AddChange(ctx)
+		_, err := GetConnectionObj(ctx, connectionID, connectionObj.AllocationID, connectionObj.ClientID)
+		if err != nil {
+			return saveChange, err
+		}
+		err = cmd.AddChange(ctx)
 		if err != nil {
 			return saveChange, err
 		}

code/go/0chain.net/blobbercore/allocation/deletefilechange.go

@@ -3,18 +3,11 @@ package allocation
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"path/filepath"
 	"sync"
 
-	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
-	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
-	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
-	"gorm.io/gorm"
-
-	"go.uber.org/zap"
 )
 
 var (
@@ -58,73 +51,8 @@ func (nf *DeleteFileChange) DeleteTempFile() error {
 	return nil
 }
 
-func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
-	db := datastore.GetStore().GetTransaction(ctx)
-	type Result struct {
-		Id               string
-		ValidationRoot   string
-		ThumbnailHash    string
-		FilestoreVersion int
-	}
-
-	limitCh := make(chan struct{}, 10)
-	wg := &sync.WaitGroup{}
-	var results []Result
-	mut.Lock()
-	err := db.Model(&reference.Ref{}).Unscoped().
-		Select("id", "validation_root", "thumbnail_hash", "filestore_version").
-		Where("allocation_id=? AND path LIKE ? AND type=? AND deleted_at is not NULL",
-			nf.AllocationID, nf.Path+"%", reference.FILE).
-		FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
-
-			for _, res := range results {
-				var count int64
-				tx.Model(&reference.Ref{}).
-					Where("allocation_id=? AND validation_root=?", nf.AllocationID, res.ValidationRoot).
-					Count(&count)
-
-				if count != 0 && res.ThumbnailHash == "" {
-					continue
-				}
-
-				limitCh <- struct{}{}
-				wg.Add(1)
-
-				go func(res Result, count int64) {
-					defer func() {
-						<-limitCh
-						wg.Done()
-					}()
-
-					if count == 0 {
-						err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ValidationRoot, res.FilestoreVersion)
-						if err != nil {
-							logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
-								zap.String("validation_root", res.ValidationRoot))
-						}
-					}
-					// We don't increase alloc size for thumbnail so we don't need to decrease it
-					// if res.ThumbnailHash != "" {
-					// 	err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ThumbnailHash)
-					// 	if err != nil {
-					// 		logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail: %s", err.Error()),
-					// 			zap.String("thumbnail", res.ThumbnailHash))
-					// 	}
-					// }
-
-				}(res, count)
-
-			}
-			return nil
-		}).Error
-	mut.Unlock()
-	wg.Wait()
-
-	return err
-	// return db.Model(&reference.Ref{}).Unscoped().
-	// 	Delete(&reference.Ref{},
-	// 		"allocation_id = ? AND path LIKE ? AND deleted_at IS NOT NULL",
-	// 		nf.AllocationID, nf.Path+"%").Error
+func (nf *DeleteFileChange) CommitToFileStore(_ context.Context, _ *sync.Mutex) error {
+	return nil
 }
 
 func (nf *DeleteFileChange) GetPath() []string {

code/go/0chain.net/blobbercore/allocation/entity.go

@@ -38,6 +38,7 @@ const (
 	CanRenameMask       = uint16(32) // 0010 0000
 )
 
+// swagger:model Allocation
 type Allocation struct {
 	ID             string           `gorm:"column:id;size:64;primaryKey"`
 	Tx             string           `gorm:"column:tx;size:64;not null;unique;index:idx_unique_allocations_tx,unique"`
@@ -207,7 +208,7 @@ func AddToPending(ctx context.Context, clientID, allocationID string, pendingWri
 	defer lock.Unlock()
 
 	pending := new(Pending)
-	err = db.Model(&Pending{}).Where("id=?", key).First(pending).Error
+	err = db.Model(&Pending{}).Where("id=?", key).Take(pending).Error
 	switch {
 	case err == nil:
 		pending.PendingWrite += pendingWrite
@@ -239,6 +240,7 @@ const (
 )
 
 // Terms for allocation by its Tx.
+// swagger:model
 type Terms struct {
 	ID           int64      `gorm:"column:id;primaryKey"`
 	BlobberID    string     `gorm:"blobber_id;size:64;not null"`