evaluation_metrics.json

{
  "test_examples": 500,
  "exact_match_accuracy": 0.026,
  "token_level_accuracy": 0.285153605015674,
  "average_similarity": 0.7674964928099656,
  "sample_predictions": [
    {
      "input": "def worker(args):\\n    env = os.environ.copy()\\n    env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME\\n   ...",
      "prediction": "def worker(args):\\n    env = os.environ.copy()\\n    env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME\\n   ...",
      "expected": "def worker(args):\\n    env = os.environ.copy()\\n    env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME\\n   ...",
      "similarity": 0.7953776775648252
    },
    {
      "input": "def login(request):\\n\\tform = forms.PasswordLoginForm()\\n\\tmagic_form = forms.EmailLoginForm()\\n\\tif...",
      "prediction": "def login(request):\\n\\tform = forms.PasswordLoginForm()\\n\\tmagic_form = forms.EmailLoginForm()\\n\\tif...",
      "expected": "def login(request):\\n\\tform = forms.PasswordLoginForm()\\n\\tmagic_form = forms.EmailLoginForm()\\n\\tif...",
      "similarity": 0.7876712328767124
    },
    {
      "input": "def CFUNCTYPE(restype, *argtypes):\\n    try:\\n        return _c_functype_cache[(restype, argtypes)]\\...",
      "prediction": "def CFUNCTYPE(restype, *argtypes):\\n    try:\\n        return _c_functype_cache[(restype, argtypes)]\\...",
      "expected": "def CFUNCTYPE(restype, *argtypes, **kw):\\n    flags = _FUNCFLAG_CDECL\\n    if kw.pop(\"use_errno\", Fa...",
      "similarity": 0.5237776289350301
    },
    {
      "input": "def _output_field_or_none(self):\\n        if self._output_field is None:\\n            self._output_f...",
      "prediction": "def _output_field_or_none(self):\\n        if self._output_field is None:\\n            return self._r...",
      "expected": "def _output_field_or_none(self):\\n        try:\\n            return self.output_field\\n        except...",
      "similarity": 0.6494252873563219
    },
    {
      "input": "def maybe_send_to_registration(\\n\\trequest: HttpRequest,\\n\\temail: str,\\n\\tfull_name: str = \"\",\\n\\tm...",
      "prediction": "def maybe_send_to_registration(\\n\\trequest: HttpRequest,\\n\\temail: str,\\n\\tfull_name: str = \"\",\\n\\tm...",
      "expected": "def maybe_send_to_registration(\\n\\trequest: HttpRequest,\\n\\temail: str,\\n\\tfull_name: str = \"\",\\n\\tm...",
      "similarity": 0.8963051251489869
    }
  ]
}