sagard21
/

python-code-explainer

text2text-generation

Trained with AutoTrain

text-generation-inference

Model card Files Files and versions

python-code-explainer / tokenizer_config.json

system's picture

system HF Staff

Commit From AutoTrain

633ef02 over 3 years ago

history blame contribute delete

1.52 kB

	{
	"add_prefix_space": false,
	"bos_token": {
	"__type": "AddedToken",
	"content": "<s>",
	"lstrip": false,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	},
	"cls_token": {
	"__type": "AddedToken",
	"content": "<s>",
	"lstrip": false,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	},
	"eos_token": {
	"__type": "AddedToken",
	"content": "</s>",
	"lstrip": false,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	},
	"errors": "replace",
	"mask_token": {
	"__type": "AddedToken",
	"content": "<mask>",
	"lstrip": true,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	},
	"model_max_length": 512,
	"name_or_path": "AutoTrain",
	"pad_token": {
	"__type": "AddedToken",
	"content": "<pad>",
	"lstrip": false,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	},
	"sep_token": {
	"__type": "AddedToken",
	"content": "</s>",
	"lstrip": false,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	},
	"special_tokens_map_file": "/export/home/cache/model/5941df5e4315c5ab63b7b2ac791fb0bf0f209744a055c06b43b5274849137cdd.b9905d0575bde443a20834122b6e2d48e853b2e36444ce98ddeb43c38097eb3f",
	"tokenizer_class": "RobertaTokenizer",
	"trim_offsets": true,
	"unk_token": {
	"__type": "AddedToken",
	"content": "<unk>",
	"lstrip": false,
	"normalized": true,
	"rstrip": false,
	"single_word": false
	}
	}