Refactor AT services (#461)

* * use AcraTranslatorData instead of fields of services,
  share all data using only AcraTranslatorData struct
* pass build flags in integrations tests
* fix overriding test in integrations tests
* make getClientID as separate function instead of method of TLSServiceWrapper
* add registries for registering callbacks on HTTP/gRPC initialization
* initialize and assign tokenizer to TranslatorData explicitly and outside of
newServer function
* extend makefile with building command of grpc services for tests
update golang services according to updates grpc/grpc-go#3669

Makefile

@@ -187,9 +187,14 @@ help:
 ## Update protobuf dependencies and regenerate .pb.go from .proto,
 ## use this target only when it is really needed, not for usual builds
 build_protobuf:
-	go get -u github.com/golang/protobuf/proto github.com/golang/protobuf/protoc-gen-go
-	go get -u google.golang.org/grpc
-	protoc --go_out=plugins=grpc:. cmd/acra-translator/grpc_api/*.proto
+	@go install google.golang.org/protobuf/cmd/[email protected]
+	@go install google.golang.org/grpc/cmd/[email protected]
+	@protoc --go_out=`pwd` --go-grpc_out=`pwd` \
+		--go_opt=module=github.com/cossacklabs/acra \
+		--go-grpc_opt=module=github.com/cossacklabs/acra \
+		-Icmd/acra-translator/grpc_api \
+		cmd/acra-translator/grpc_api/*.proto
+	@python3 -m grpc_tools.protoc -Icmd/acra-translator/grpc_api --proto_path=. --python_out=tests/ --grpc_python_out=tests/ cmd/acra-translator/grpc_api/*.proto
 
 ## Build the application in the subdirectory (default)
 build:

cmd/acra-translator/acra-translator.go

@@ -42,6 +42,7 @@ import (
 	filesystemBackendV2CE "github.com/cossacklabs/acra/keystore/v2/keystore/filesystem/backend"
 	"github.com/cossacklabs/acra/logging"
 	"github.com/cossacklabs/acra/network"
+	"github.com/cossacklabs/acra/poison"
 	"github.com/cossacklabs/acra/pseudonymization"
 	common2 "github.com/cossacklabs/acra/pseudonymization/common"
 	"github.com/cossacklabs/acra/pseudonymization/storage"
@@ -377,15 +378,34 @@ func realMain() error {
 		return err
 	}
 	config.SetTokenizer(tokenizer)
-	serviceFactory, err := grpc_api.NewgRPCServerFactory(tokenizer, keyStore)
+	poisonCallbacks := poison.NewCallbackStorage()
+	if config.DetectPoisonRecords() {
+		// used to turn off poison record detection which rely on HasCallbacks
+		poisonCallbacks.AddCallback(poison.EmptyCallback{})
+		if config.ScriptOnPoison() != "" {
+			poisonCallbacks.AddCallback(poison.NewExecuteScriptCallback(config.ScriptOnPoison()))
+		}
+		// should setup "stopOnPoison" as last poison record callback"
+		if config.StopOnPoison() {
+			poisonCallbacks.AddCallback(&poison.StopCallback{})
+		}
+	}
+	translatorData := &common.TranslatorData{
+		Tokenizer:             tokenizer,
+		Config:                config,
+		Keystorage:            transportKeystore,
+		PoisonRecordCallbacks: poisonCallbacks,
+		UseConnectionClientID: config.GetUseClientIDFromConnection(),
+		TLSClientIDExtractor:  config.GetTLSClientIDExtractor(),
+	}
+	grpcServer, err := grpc_api.NewServer(translatorData, config.GRPCConnectionWrapper)
 	if err != nil {
 		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorGeneral).Errorln("Can't initialize gRPC service factory")
 		return err
 	}
 
-	var readerServer *server.ReaderServer
 	waitTimeout := time.Duration(*closeConnectionTimeout) * time.Second
-	readerServer, err = server.NewReaderServer(config, transportKeystore, serviceFactory, waitTimeout)
+	readerServer, err := server.NewReaderServer(translatorData, grpcServer, waitTimeout)
 	if err != nil {
 		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantStartService).
 			Errorf("System error: can't start %s", ServiceName)

cmd/acra-translator/common/common.go

@@ -21,10 +21,13 @@ import (
 	"github.com/cossacklabs/acra/decryptor/base"
 	"github.com/cossacklabs/acra/keystore"
 	"github.com/cossacklabs/acra/network"
+	tokenCommon "github.com/cossacklabs/acra/pseudonymization/common"
 )
 
 // TranslatorData connects KeyStorage and Poison records settings for HTTP and gRPC decryptors.
 type TranslatorData struct {
+	Tokenizer             tokenCommon.Pseudoanonymizer
+	Config                *AcraTranslatorConfig
 	PoisonRecordCallbacks base.PoisonRecordCallbackStorage
 	Keystorage            keystore.TranslationKeyStore
 	UseConnectionClientID bool

cmd/acra-translator/common/service.go

@@ -29,13 +29,12 @@ type ITranslatorService interface {
 // TranslatorService service that implements all Acra-Translator functions
 type TranslatorService struct {
 	data           *TranslatorData
-	tokenizer      tokenCommon.Pseudoanonymizer
 	handler        crypto.RegistryHandler
 	poisonDetector *crypto.EnvelopeDetector
 }
 
 // NewTranslatorService return new initialized TranslatorService
-func NewTranslatorService(translatorData *TranslatorData, tokenizer tokenCommon.Pseudoanonymizer) (*TranslatorService, error) {
+func NewTranslatorService(translatorData *TranslatorData) (*TranslatorService, error) {
 	registryHandler := crypto.NewRegistryHandler(translatorData.Keystorage)
 	poisonEnvelopeDetector := crypto.NewEnvelopeDetector()
 	if translatorData.PoisonRecordCallbacks != nil && translatorData.PoisonRecordCallbacks.HasCallbacks() {
@@ -44,7 +43,7 @@ func NewTranslatorService(translatorData *TranslatorData, tokenizer tokenCommon.
 		poisonDetector.SetPoisonRecordCallbacks(translatorData.PoisonRecordCallbacks)
 		poisonEnvelopeDetector.AddCallback(poisonDetector)
 	}
-	return &TranslatorService{data: translatorData, tokenizer: tokenizer, handler: registryHandler, poisonDetector: poisonEnvelopeDetector}, nil
+	return &TranslatorService{data: translatorData, handler: registryHandler, poisonDetector: poisonEnvelopeDetector}, nil
 }
 
 // Errors possible during decrypting AcraStructs.
@@ -257,7 +256,7 @@ func (service *TranslatorService) Tokenize(ctx context.Context, data interface{}
 	if len(zoneID) > 0 {
 		tokenContext = tokenCommon.TokenContext{ZoneID: zoneID}
 	}
-	response, err := service.tokenizer.AnonymizeConsistently(data, tokenContext, dataType)
+	response, err := service.data.Tokenizer.AnonymizeConsistently(data, tokenContext, dataType)
 	if err != nil {
 		logger.WithError(err).Errorln("Can't tokenize")
 		return nil, ErrTokenize
@@ -277,7 +276,7 @@ func (service *TranslatorService) Detokenize(ctx context.Context, data interface
 	}
 	switch dataType {
 	case tokenCommon.TokenType_Bytes, tokenCommon.TokenType_Email, tokenCommon.TokenType_Int32, tokenCommon.TokenType_Int64, tokenCommon.TokenType_String:
-		sourceData, err := service.tokenizer.Deanonymize(data, tokenContext, dataType)
+		sourceData, err := service.data.Tokenizer.Deanonymize(data, tokenContext, dataType)
 		if err != nil {
 			logger.WithField("type", dataType).WithError(err).Errorln("Can't tokenize data")
 			return nil, ErrDetokenize