{"package":"r-tokenizers","ecosystem":"conda","latest_version":"0.3.0","description":"Convert natural language text into tokens. Includes tokenizers for shingled n-grams, skip n-grams, words, word stems, sentences, paragraphs, characters, shingled characters, lines, tweets, Penn Treebank, regular expressions, as well as functions for counting characters, words, and sentences, and a function for splitting longer texts into separate documents, each with the same number of words.  The tokenizers have a consistent interface, and the package is built on the 'stringi' and 'Rcpp' packages for  fast yet correct tokenization in 'UTF-8'.","license":"MIT","license_risk":"permissive","commercial_use_notes":"Permissive: commercial closed-source use OK; preserve the copyright notice.","homepage":"https://lincolnmullen.com/software/tokenizers/","repository":"","downloads_weekly":510,"health":{"score":52,"risk":"high","breakdown":{"maintenance":10,"popularity":3,"security":25,"maturity":12,"community":2},"deprecated":false,"max_score":100},"vulnerabilities":{"count":0,"critical":0,"high":0,"medium":0,"low":0,"details":[]},"versions":{"latest":"0.3.0","total_count":4,"recent":["0.1.4","0.2.1","0.2.3","0.3.0"]},"metadata":{"deprecated":false,"deprecated_message":null,"maintainers_count":1,"first_published":"2020-07-14 12:02:53.225000+00:00","last_published":"2025-09-12 05:31:04.879000+00:00","dependencies_count":0,"dependencies":[]},"github_stats":null,"bundle":null,"typescript":null,"known_issues":{"bugs_count":0,"bugs_severity":{},"status_breakdown":{},"link":null,"scope":"none"},"historical_compromise":null,"recommendation":{"action":"safe_to_use","issues":[],"use_version":"0.3.0","version_hint":null,"summary":"r-tokenizers@0.3.0 is safe to use (health: 52/100)"},"version_scoped":null,"requested_version":null,"_cache":"miss","_response_ms":586,"_powered_by":"depscope.dev — free package intelligence for AI agents","typosquat":{"is_suspected":false},"maintainer_trust":{"available":false},"malicious":{"is_malicious":false},"scorecard":{"available":false},"quality":{"available":false},"version_history_summary":{"total_versions":4,"first_release_age_days":2115,"last_release_days_ago":230,"avg_days_between_releases":705,"release_velocity":"moderate"}}