1 //===- TFUtilsTest.cpp - test for TFUtils ---------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Analysis/Utils/TFUtils.h" 10 #include "google/protobuf/struct.pb.h" 11 #include "tensorflow/core/example/example.pb.h" 12 #include "tensorflow/core/example/feature.pb.h" 13 #include "llvm/AsmParser/Parser.h" 14 #include "llvm/IR/Dominators.h" 15 #include "llvm/IR/Instructions.h" 16 #include "llvm/IR/LLVMContext.h" 17 #include "llvm/IR/Module.h" 18 #include "llvm/Support/Path.h" 19 #include "llvm/Support/SourceMgr.h" 20 #include "llvm/Testing/Support/SupportHelpers.h" 21 #include "gtest/gtest.h" 22 23 using namespace llvm; 24 25 extern const char *TestMainArgv0; 26 27 // NOTE! This test model is currently also used by test/Transforms/Inline/ML tests 28 //- relevant if updating this model. 29 static std::string getModelPath() { 30 SmallString<128> InputsDir = unittest::getInputFileDirectory(TestMainArgv0); 31 llvm::sys::path::append(InputsDir, "ir2native_x86_64_model"); 32 return std::string(InputsDir); 33 } 34 35 // Test observable behavior when no model is provided. 36 TEST(TFUtilsTest, NoModel) { 37 TFModelEvaluator Evaluator("", {}, {}); 38 EXPECT_FALSE(Evaluator.isValid()); 39 } 40 41 // Test we can correctly load a savedmodel and evaluate it. 42 TEST(TFUtilsTest, LoadAndExecuteTest) { 43 // We use the ir2native model for test. We know it has one feature of 44 // dimension (1, 214) 45 const static int64_t KnownSize = 214; 46 std::vector<TensorSpec> InputSpecs{TensorSpec::createSpec<int32_t>( 47 "serving_default_input_1", {1, KnownSize})}; 48 std::vector<TensorSpec> OutputSpecs{ 49 TensorSpec::createSpec<float>("StatefulPartitionedCall", {1})}; 50 51 TFModelEvaluator Evaluator(getModelPath(), InputSpecs, OutputSpecs); 52 EXPECT_TRUE(Evaluator.isValid()); 53 54 int32_t *V = Evaluator.getInput<int32_t>(0); 55 // Fill it up with 1's, we know the output. 56 for (auto I = 0; I < KnownSize; ++I) { 57 V[I] = 1; 58 } 59 { 60 auto ER = Evaluator.evaluate(); 61 EXPECT_TRUE(ER.hasValue()); 62 float Ret = *ER->getTensorValue<float>(0); 63 EXPECT_EQ(static_cast<int64_t>(Ret), 80); 64 EXPECT_EQ(ER->getUntypedTensorValue(0), 65 reinterpret_cast<const void *>(ER->getTensorValue<float>(0))); 66 } 67 // The input vector should be unchanged 68 for (auto I = 0; I < KnownSize; ++I) { 69 EXPECT_EQ(V[I], 1); 70 } 71 // Zero-out the unused position '0' of the instruction histogram, which is 72 // after the first 9 calculated values. Should the the same result. 73 V[9] = 0; 74 { 75 auto ER = Evaluator.evaluate(); 76 EXPECT_TRUE(ER.hasValue()); 77 float Ret = *ER->getTensorValue<float>(0); 78 EXPECT_EQ(static_cast<int64_t>(Ret), 80); 79 } 80 } 81 82 // Test incorrect input setup 83 TEST(TFUtilsTest, EvalError) { 84 // We use the ir2native model for test. We know it has one feature of 85 // dimension (1, 214) 86 const static int64_t KnownSize = 213; 87 std::vector<TensorSpec> InputSpecs{TensorSpec::createSpec<int32_t>( 88 "serving_default_input_1", {1, KnownSize})}; 89 std::vector<TensorSpec> OutputSpecs{ 90 TensorSpec::createSpec<float>("StatefulPartitionedCall", {1})}; 91 92 TFModelEvaluator Evaluator(getModelPath(), InputSpecs, OutputSpecs); 93 EXPECT_TRUE(Evaluator.isValid()); 94 95 int32_t *V = Evaluator.getInput<int32_t>(0); 96 // Fill it up with 1's, we know the output. 97 for (auto I = 0; I < KnownSize; ++I) { 98 V[I] = 1; 99 } 100 auto ER = Evaluator.evaluate(); 101 EXPECT_FALSE(ER.hasValue()); 102 EXPECT_FALSE(Evaluator.isValid()); 103 } 104 105 #define PROTO_CHECKER(FNAME, TYPE, INDEX, EXP) \ 106 do { \ 107 const auto &V = Expected.feature_lists() \ 108 .feature_list() \ 109 .at(FNAME) \ 110 .feature(INDEX) \ 111 .TYPE() \ 112 .value(); \ 113 for (auto I = 0; I < V.size(); ++I) \ 114 EXPECT_EQ(V.at(I), EXP[I]); \ 115 } while (false) 116 117 TEST(TFUtilsTest, Logger) { 118 std::vector<LoggedFeatureSpec> Features; 119 Features.push_back( 120 {TensorSpec::createSpec<float>("the_float", {2, 3}), None}); 121 Features.push_back({TensorSpec::createSpec<int64_t>("the_int", {2}), 122 std::string("alternate_name")}); 123 124 auto Rewards = TensorSpec::createSpec<float>("reward", {1}); 125 Logger L(Features, Rewards, true); 126 const float F00[]{0.0, 0.1, 0.2, 0.3, 0.4, 0.5}; 127 const int64_t F01[]{2, 3}; 128 129 L.logFloatValue(0, F00); 130 L.logInt64Value(1, F01); 131 L.logFloatReward(3.4); 132 const float F10[]{0.0, 1.0, 2.0, 3.0, 4.0, 5.0}; 133 const int64_t F11[]{-2, -3}; 134 L.logFloatValue(0, F10); 135 L.logInt64Value(1, F11); 136 L.logFloatReward(-3.0); 137 std::string Result; 138 raw_string_ostream OS(Result); 139 L.flush(OS); 140 141 tensorflow::SequenceExample Expected; 142 ASSERT_TRUE(Expected.ParseFromString(Result)); 143 PROTO_CHECKER("the_float", float_list, 0, F00); 144 PROTO_CHECKER("the_float", float_list, 1, F10); 145 PROTO_CHECKER("alternate_name", int64_list, 0, F01); 146 PROTO_CHECKER("alternate_name", int64_list, 1, F11); 147 float R0[]{3.4}; 148 float R1[]{-3.0}; 149 PROTO_CHECKER("reward", float_list, 0, R0); 150 PROTO_CHECKER("reward", float_list, 1, R1); 151 } 152 153 TEST(TFUtilsTest, LoggerInt32FeaturesAndReward) { 154 std::vector<LoggedFeatureSpec> Features; 155 Features.push_back( 156 {TensorSpec::createSpec<float>("the_float", {2, 3}), None}); 157 Features.push_back({TensorSpec::createSpec<int32_t>("the_int", {2}), 158 std::string("alternate_name")}); 159 160 auto Rewards = TensorSpec::createSpec<int32_t>("reward", {1}); 161 Logger L(Features, Rewards, true); 162 const float F00[]{0.0, 0.1, 0.2, 0.3, 0.4, 0.5}; 163 const int32_t F01[]{2, 3}; 164 165 L.logFloatValue(0, F00); 166 L.logInt32Value(1, F01); 167 L.logInt32Reward(3); 168 const float F10[]{0.0, 1.0, 2.0, 3.0, 4.0, 5.0}; 169 const int32_t F11[]{-2, -3}; 170 L.logFloatValue(0, F10); 171 L.logInt32Value(1, F11); 172 L.logInt32Reward(-3); 173 std::string Result; 174 raw_string_ostream OS(Result); 175 L.flush(OS); 176 177 tensorflow::SequenceExample Expected; 178 ASSERT_TRUE(Expected.ParseFromString(Result)); 179 PROTO_CHECKER("the_float", float_list, 0, F00); 180 PROTO_CHECKER("the_float", float_list, 1, F10); 181 PROTO_CHECKER("alternate_name", int64_list, 0, F01); 182 PROTO_CHECKER("alternate_name", int64_list, 1, F11); 183 int32_t R0[]{3}; 184 int32_t R1[]{-3}; 185 PROTO_CHECKER("reward", int64_list, 0, R0); 186 PROTO_CHECKER("reward", int64_list, 1, R1); 187 } 188 189 TEST(TFUtilsTest, LoggerNoReward) { 190 std::vector<LoggedFeatureSpec> Features; 191 Features.push_back( 192 {TensorSpec::createSpec<float>("the_float", {2, 3}), None}); 193 Features.push_back({TensorSpec::createSpec<int64_t>("the_int", {2}), 194 std::string("alternate_name")}); 195 196 auto Rewards = TensorSpec::createSpec<float>("reward", {1}); 197 Logger L(Features, Rewards, false); 198 const float F00[]{0.0, 0.1, 0.2, 0.3, 0.4, 0.5}; 199 const int64_t F01[]{2, 3}; 200 201 L.logFloatValue(0, F00); 202 L.logInt64Value(1, F01); 203 const float F10[]{0.0, 1.0, 2.0, 3.0, 4.0, 5.0}; 204 const int64_t F11[]{-2, -3}; 205 L.logFloatValue(0, F10); 206 L.logInt64Value(1, F11); 207 208 std::string Result; 209 raw_string_ostream OS(Result); 210 L.flush(OS); 211 tensorflow::SequenceExample Expected; 212 ASSERT_TRUE(Expected.ParseFromString(Result)); 213 PROTO_CHECKER("the_float", float_list, 0, F00); 214 PROTO_CHECKER("the_float", float_list, 1, F10); 215 PROTO_CHECKER("alternate_name", int64_list, 0, F01); 216 PROTO_CHECKER("alternate_name", int64_list, 1, F11); 217 } 218 219 TEST(TFUtilsTest, LoggerFinalReward) { 220 std::vector<LoggedFeatureSpec> Features; 221 Features.push_back({TensorSpec::createSpec<float>("the_float", {1}), None}); 222 Features.push_back({TensorSpec::createSpec<int64_t>("the_int", {1}), None}); 223 224 auto Rewards = TensorSpec::createSpec<float>("reward", {1}); 225 Logger L(Features, Rewards, true); 226 for (int64_t I = 0; I < 3; ++I) { 227 float F = static_cast<float>(I); 228 L.logFloatValue(0, &F); 229 L.logInt64Value(1, &I); 230 } 231 L.logFloatFinalReward(3.14); 232 std::string Result; 233 raw_string_ostream OS(Result); 234 L.flush(OS); 235 const float Zero[]{0.0}; 236 const float R[]{3.14}; 237 tensorflow::SequenceExample Expected; 238 ASSERT_TRUE(Expected.ParseFromString(Result)); 239 PROTO_CHECKER("reward", float_list, 0, Zero); 240 PROTO_CHECKER("reward", float_list, 1, Zero); 241 PROTO_CHECKER("reward", float_list, 2, R); 242 } 243 244 TEST(TFUtilsTest, LoggerGroup) { 245 std::vector<LoggedFeatureSpec> Features; 246 Features.push_back({TensorSpec::createSpec<float>("the_float", {1}), None}); 247 Features.push_back({TensorSpec::createSpec<int64_t>("the_int", {1}), None}); 248 249 auto Rewards = TensorSpec::createSpec<float>("reward", {1}); 250 StringMap<std::unique_ptr<Logger>> Loggers; 251 std::vector<std::string> Names{"a", "b"}; 252 size_t Bump = 0; 253 for (auto Name : Names) { 254 auto L = std::make_unique<Logger>(Features, Rewards, true); 255 for (int64_t I = 0; I < 3; ++I) { 256 float F = static_cast<float>(I) + Bump; 257 L->logFloatValue(0, &F); 258 L->logInt64Value(1, &I); 259 } 260 L->logFloatFinalReward(3.14 + Bump); 261 Loggers.insert(std::make_pair(Name, std::move(L))); 262 } 263 std::string Result; 264 raw_string_ostream OS(Result); 265 Logger::flushLogs(OS, Loggers); 266 google::protobuf::Struct Expected; 267 ASSERT_TRUE(Expected.ParseFromString(Result)); 268 EXPECT_EQ(Expected.fields_size(), 2); 269 EXPECT_TRUE(Expected.fields().contains("a")); 270 EXPECT_TRUE(Expected.fields().contains("b")); 271 } 272