tensorflow serving java案例

背景介绍

这篇文章是tensorflow serving java api使用的参考案例，基本上把TFS的核心API的用法都介绍清楚。案例主要分为三部分：

• 动态更新模型：用于在TFS处于runtime时候动态加载模型。
• 获取模型状态：用于获取加载的模型的基本信息。
• 在线模型预测：进行在线预测，分类等操作，着重介绍在线预测。

因为模型的预测需要参考模型内部变量，所以可以先行通过TFS的REST接口获取TF模型的元数据然后才能构建TFS的RPC请求对象。

TFS 使用入门

模型源数据获取

curl http://host:port/v1/models/${MODEL_NAME}[/versions/${MODEL_VERSION}]/metadata

说明：

• 参考TFS REST API
• 返回结果参考TF模型结构。

 public static void getModelStatus() { // 1、设置访问的RPC协议的host和port ManagedChannel channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); // 2、构建PredictionServiceBlockingStub对象 PredictionServiceGrpc.PredictionServiceBlockingStub predictionServiceBlockingStub = PredictionServiceGrpc.newBlockingStub(channel); // 3、设置待获取的模型 Model.ModelSpec modelSpec = Model.ModelSpec.newBuilder() .setName("wdl_model").build(); // 4、构建获取元数据的请求 GetModelMetadata.GetModelMetadataRequest modelMetadataRequest = GetModelMetadata.GetModelMetadataRequest.newBuilder() .setModelSpec(modelSpec) .addAllMetadataField(Arrays.asList("signature_def")) .build(); // 5、获取元数据 GetModelMetadata.GetModelMetadataResponse getModelMetadataResponse = predictionServiceBlockingStub.getModelMetadata(modelMetadataRequest); channel.shutdownNow(); }

说明：

• Model.ModelSpec.newBuilder绑定需要访问的模型的名字。
• GetModelMetadataRequest中addAllMetadataField绑定curl命令返回的metadata当中的signature_def 字段。

动态更新模型

 public static void addNewModel() { // 1、构建动态更新模型1 ModelServerConfigOuterClass.ModelConfig modelConfig1 = ModelServerConfigOuterClass.ModelConfig.newBuilder() .setBasePath("/models/new_model") .setName("new_model") .setModelType(ModelServerConfigOuterClass.ModelType.TENSORFLOW) .build(); // 2、构建动态更新模型2 ModelServerConfigOuterClass.ModelConfig modelConfig2 = ModelServerConfigOuterClass.ModelConfig.newBuilder() .setBasePath("/models/wdl_model") .setName("wdl_model") .setModelType(ModelServerConfigOuterClass.ModelType.TENSORFLOW) .build(); // 3、合并动态更新模型到ModelConfigList对象中 ModelServerConfigOuterClass.ModelConfigList modelConfigList = ModelServerConfigOuterClass.ModelConfigList.newBuilder() .addConfig(modelConfig1) .addConfig(modelConfig2) .build(); // 4、添加到ModelConfigList到ModelServerConfig对象当中 ModelServerConfigOuterClass.ModelServerConfig modelServerConfig = ModelServerConfigOuterClass.ModelServerConfig.newBuilder() .setModelConfigList(modelConfigList) .build(); // 5、构建ReloadConfigRequest并绑定ModelServerConfig对象。 ModelManagement.ReloadConfigRequest reloadConfigRequest = ModelManagement.ReloadConfigRequest.newBuilder() .setConfig(modelServerConfig) .build(); // 6、构建modelServiceBlockingStub访问句柄 ManagedChannel channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); ModelServiceGrpc.ModelServiceBlockingStub modelServiceBlockingStub = ModelServiceGrpc.newBlockingStub(channel); ModelManagement.ReloadConfigResponse reloadConfigResponse = modelServiceBlockingStub.handleReloadConfigRequest(reloadConfigRequest); System.out.println(reloadConfigResponse.getStatus().getErrorMessage()); channel.shutdownNow(); }

说明：

• 动态更新模型是一个全量的模型加载，在发布A模型后想动态发布B模型需要同时传递模型A和B的信息。
• 再次强调，需要全量更新，全量更新，全量更新！！！

在线模型预测

 public static void doPredict() throws Exception { // 1、构建feature Map<String, Feature> featureMap = new HashMap<>(); featureMap.put("match_type", feature("")); featureMap.put("position", feature(0.0f)); featureMap.put("brand_prefer_1d", feature(0.0f)); featureMap.put("brand_prefer_1m", feature(0.0f)); featureMap.put("brand_prefer_1w", feature(0.0f)); featureMap.put("brand_prefer_2w", feature(0.0f)); featureMap.put("browse_norm_score_1d", feature(0.0f)); featureMap.put("browse_norm_score_1w", feature(0.0f)); featureMap.put("browse_norm_score_2w", feature(0.0f)); featureMap.put("buy_norm_score_1d", feature(0.0f)); featureMap.put("buy_norm_score_1w", feature(0.0f)); featureMap.put("buy_norm_score_2w", feature(0.0f)); featureMap.put("cate1_prefer_1d", feature(0.0f)); featureMap.put("cate1_prefer_2d", feature(0.0f)); featureMap.put("cate1_prefer_1m", feature(0.0f)); featureMap.put("cate1_prefer_1w", feature(0.0f)); featureMap.put("cate1_prefer_2w", feature(0.0f)); featureMap.put("cate2_prefer_1d", feature(0.0f)); featureMap.put("cate2_prefer_1m", feature(0.0f)); featureMap.put("cate2_prefer_1w", feature(0.0f)); featureMap.put("cate2_prefer_2w", feature(0.0f)); featureMap.put("cid_prefer_1d", feature(0.0f)); featureMap.put("cid_prefer_1m", feature(0.0f)); featureMap.put("cid_prefer_1w", feature(0.0f)); featureMap.put("cid_prefer_2w", feature(0.0f)); featureMap.put("user_buy_rate_1d", feature(0.0f)); featureMap.put("user_buy_rate_2w", feature(0.0f)); featureMap.put("user_click_rate_1d", feature(0.0f)); featureMap.put("user_click_rate_1w", feature(0.0f)); Features features = Features.newBuilder().putAllFeature(featureMap).build(); Example example = Example.newBuilder().setFeatures(features).build(); // 2、构建Predict请求 Predict.PredictRequest.Builder predictRequestBuilder = Predict.PredictRequest.newBuilder(); // 3、构建模型请求维度ModelSpec，绑定模型名和预测的签名 Model.ModelSpec.Builder modelSpecBuilder = Model.ModelSpec.newBuilder(); modelSpecBuilder.setName("wdl_model"); modelSpecBuilder.setSignatureName("predict"); predictRequestBuilder.setModelSpec(modelSpecBuilder); // 4、构建预测请求的维度信息DIM对象 TensorShapeProto.Dim dim = TensorShapeProto.Dim.newBuilder().setSize(300).build(); TensorShapeProto shapeProto = TensorShapeProto.newBuilder().addDim(dim).build(); TensorProto.Builder tensor = TensorProto.newBuilder(); tensor.setTensorShape(shapeProto); tensor.setDtype(DataType.DT_STRING); // 5、批量绑定预测请求的数据 for (int i=0; i<300; i++) { tensor.addStringVal(example.toByteString()); } predictRequestBuilder.putInputs("examples", tensor.build()); // 6、构建PredictionServiceBlockingStub对象准备预测 ManagedChannel channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); PredictionServiceGrpc.PredictionServiceBlockingStub predictionServiceBlockingStub = PredictionServiceGrpc.newBlockingStub(channel); // 7、执行预测 Predict.PredictResponse predictResponse = predictionServiceBlockingStub.predict(predictRequestBuilder.build()); // 8、解析请求结果 List<Float> floatList = predictResponse .getOutputsOrThrow("probabilities") .getFloatValList(); }

说明：

• TFS的RPC请求过程中设置的参数需要考虑TF模型的数据结构。
• TFS的RPC请求有同步和异步两种方式，上述只展示同步方式。

TF模型结构

{ "model_spec": { "name": "wdl_model", "signature_name": "", "version": "4" }, "metadata": { "signature_def": { "signature_def": { "predict": { "inputs": { "examples": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }], "unknown_rank": false }, "name": "input_example_tensor:0" } }, "outputs": { "logistic": { "dtype": "DT_FLOAT", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "1", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/logistic:0" }, "class_ids": { "dtype": "DT_INT64", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "1", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/ExpandDims:0" }, "probabilities": { "dtype": "DT_FLOAT", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "2", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/probabilities:0" }, "classes": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "1", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/str_classes:0" }, "logits": { "dtype": "DT_FLOAT", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "1", "name": "" } ], "unknown_rank": false }, "name": "add:0" } }, "method_name": "tensorflow/serving/predict" }, "classification": { "inputs": { "inputs": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }], "unknown_rank": false }, "name": "input_example_tensor:0" } }, "outputs": { "classes": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "2", "name": "" } ], "unknown_rank": false }, "name": "head/Tile:0" }, "scores": { "dtype": "DT_FLOAT", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "2", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/probabilities:0" } }, "method_name": "tensorflow/serving/classify" }, "regression": { "inputs": { "inputs": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }], "unknown_rank": false }, "name": "input_example_tensor:0" } }, "outputs": { "outputs": { "dtype": "DT_FLOAT", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "1", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/logistic:0" } }, "method_name": "tensorflow/serving/regress" }, "serving_default": { "inputs": { "inputs": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }], "unknown_rank": false }, "name": "input_example_tensor:0" } }, "outputs": { "classes": { "dtype": "DT_STRING", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "2", "name": "" } ], "unknown_rank": false }, "name": "head/Tile:0" }, "scores": { "dtype": "DT_FLOAT", "tensor_shape": { "dim": [{ "size": "-1", "name": "" }, { "size": "2", "name": "" } ], "unknown_rank": false }, "name": "head/predictions/probabilities:0" } }, "method_name": "tensorflow/serving/classify" } } } } }