Takara-DS1/miru-codev3-tokenlearn
Feature Extraction • 21M • Updated
text stringlengths 11 6.3k | embedding listlengths 768 768 |
|---|---|
func detectGoPackageForProject(projectFile string) (string, error) {
var goPkg string
projectDir := filepath.Dir(projectFile)
if err := filepath.Walk(projectDir, func(protoFile string, info os.FileInfo, err error) error {
// already set
if goPkg != "" {
return nil
}
if !strings.HasSuffix(protoFile, ".prot... | [
0.33890026807785034,
-0.016921069473028183,
0.8212646245956421,
0.1626608818769455,
0.21889296174049377,
0.23448097705841064,
0.1412288248538971,
-0.1430421769618988,
-0.7737232446670532,
0.4865230321884155,
0.3493833541870117,
-0.3175307512283325,
-0.5887952446937561,
0.6279798746109009,
... |
func NewQueueManager(name string, clusters []string, aliasQueues []AliasQueue, remoteQueues []RemoteQueue, clusterQueues []ClusterQueue, ) *QueueManager {
this := QueueManager{}
this.Name = name
this.Clusters = clusters
this.AliasQueues = aliasQueues
this.RemoteQueues = remoteQueues
this.ClusterQueues = clusterQu... | [
-0.3267722427845001,
-0.9674072265625,
0.4978991746902466,
-0.6866825222969055,
-0.20668165385723114,
-0.1689014583826065,
-1.331803560256958,
-0.023207658901810646,
0.5008798837661743,
0.1812879890203476,
-0.7106329202651978,
0.37847790122032166,
-0.13039086759090424,
1.4233715534210205,
... |
func NewQueueManagerWithDefaults() *QueueManager {
this := QueueManager{}
return &this
} | [
-0.4501911401748657,
-0.5164682865142822,
-0.016295745968818665,
0.26893872022628784,
0.4867601990699768,
-0.9880926012992859,
-1.3092403411865234,
-0.769955575466156,
0.6591994762420654,
-1.0082403421401978,
-0.8720371723175049,
-1.0555191040039062,
-0.5024347305297852,
1.6800291538238525... |
func (o *QueueManager) GetName() string {
if o == nil {
var ret string
return ret
}
return o.Name
} | [
0.2783944010734558,
-0.4653491973876953,
0.07367941737174988,
0.3524421155452728,
-0.2706078290939331,
-0.14447839558124542,
0.46778374910354614,
-0.4549126923084259,
0.19268442690372467,
0.20512324571609497,
-0.24332362413406372,
-0.9114316701889038,
-0.07411373406648636,
0.72692352533340... |
func (o *QueueManager) GetNameOk() (*string, bool) {
if o == nil {
return nil, false
}
return &o.Name, true
} | [
-0.06180238723754883,
-0.1152486801147461,
0.7517033815383911,
0.22422705590724945,
-0.6773747205734253,
0.5031868815422058,
0.04231294244527817,
-0.442088782787323,
-0.32960960268974304,
1.1810302734375,
-1.5875226259231567,
-0.8380942940711975,
0.30232009291648865,
0.5984615683555603,
... |
func (o *QueueManager) SetName(v string) {
o.Name = v
} | [
-0.06401383876800537,
-0.608516275882721,
0.031669050455093384,
-0.17581892013549805,
-1.1921958923339844,
-0.23478369414806366,
-0.5268489718437195,
-0.475983589887619,
0.051118120551109314,
0.7813145518302917,
-1.1876726150512695,
-0.02378077432513237,
0.07162251323461533,
0.999849200248... |
func (o *QueueManager) GetClusters() []string {
if o == nil {
var ret []string
return ret
}
return o.Clusters
} | [
-0.013011036440730095,
0.17286595702171326,
0.13941864669322968,
0.8525431156158447,
0.5821910500526428,
0.17180776596069336,
0.5408411622047424,
-1.1775822639465332,
1.2082362174987793,
-0.48590192198753357,
0.4208449125289917,
-0.3377595543861389,
-0.8340001702308655,
0.25026997923851013... |
func (o *QueueManager) GetClustersOk() (*[]string, bool) {
if o == nil {
return nil, false
}
return &o.Clusters, true
} | [
-0.3407321274280548,
0.43987908959388733,
0.28302431106567383,
0.6448001861572266,
-0.23482391238212585,
-0.15155042707920074,
0.9290333986282349,
-0.38941386342048645,
-0.14202608168125153,
0.7272371053695679,
-1.0119786262512207,
-0.8653504252433777,
-0.47515028715133667,
0.2443217933177... |
func (o *QueueManager) SetClusters(v []string) {
o.Clusters = v
} | [
-0.35853952169418335,
0.12296758592128754,
-0.08707837015390396,
0.1853027045726776,
-0.051385678350925446,
0.32632842659950256,
0.02679133228957653,
-1.1720556020736694,
0.3224264085292816,
0.23002229630947113,
-0.22711113095283508,
0.09273386001586914,
-1.0973548889160156,
0.573125839233... |
func (o *QueueManager) GetAliasQueues() []AliasQueue {
if o == nil {
var ret []AliasQueue
return ret
}
return o.AliasQueues
} | [
1.031572699546814,
-1.2376437187194824,
0.5159135460853577,
0.5704042911529541,
0.216475710272789,
0.8114396333694458,
-0.16581901907920837,
0.5874223113059998,
0.49837175011634827,
0.04552581161260605,
0.20500487089157104,
-0.156958669424057,
-0.23396286368370056,
1.5062204599380493,
0.... |
func (o *QueueManager) GetAliasQueuesOk() (*[]AliasQueue, bool) {
if o == nil {
return nil, false
}
return &o.AliasQueues, true
} | [
0.3174998462200165,
-1.141539216041565,
0.32761892676353455,
-0.09985054284334183,
-0.43492263555526733,
0.4578562080860138,
0.08670713007450104,
0.1154901310801506,
-0.14358124136924744,
1.0867294073104858,
-0.8186934590339661,
-1.042433261871338,
0.8512003421783447,
0.7666493058204651,
... |
func (o *QueueManager) SetAliasQueues(v []AliasQueue) {
o.AliasQueues = v
} | [
0.2639486491680145,
-1.250320553779602,
0.19162698090076447,
-0.11469986289739609,
-0.34679752588272095,
0.7015591859817505,
-0.30842915177345276,
0.06785077601671219,
0.1256706267595291,
0.8842712640762329,
-0.23299415409564972,
-0.10200185328722,
-0.09571540355682373,
1.8979580402374268,... |
End of preview. Expand in Data Studio
minishlab/tokenlearn-cornstack-docs-coderankembed-v2 Dataset Card
This dataset was created with Tokenlearn for training Model2Vec models on code retrieval. It contains mean token embeddings produced by nomic-ai/CodeRankEmbed, used as training targets for static embedding distillation.
The dataset contains code documents from CornStack across 6 programming languages (100,000 rows per language, 600,000 total).
Dataset Details
| Field | Value |
|---|---|
| Source | CornStack (nomic-ai) |
| Embedding model | nomic-ai/CodeRankEmbed |
| Embedding dimension | 768 |
| Languages | Python, Java, PHP, Go, JavaScript, Ruby |
| Rows per language | 100,000 |
| Total rows | 600,000 |
| Field | document |
Source Datasets
| Language | Source |
|---|---|
python |
nomic-ai/cornstack-python-v1 |
java |
nomic-ai/cornstack-java-v1 |
php |
nomic-ai/cornstack-php-v1 |
go |
nomic-ai/cornstack-go-v1 |
javascript |
nomic-ai/cornstack-javascript-v1 |
ruby |
nomic-ai/cornstack-ruby-v1 |
Dataset Structure
| Column | Type | Description |
|---|---|---|
text |
string |
Truncated input text (tokenizer max length 512) |
embedding |
list[float32] |
Mean token embedding from nomic-ai/CodeRankEmbed, excluding BOS/EOS tokens |
Usage
Load a single language config:
from datasets import load_dataset
# Load Python code documents
dataset = load_dataset("minishlab/tokenlearn-cornstack-docs-coderankembed", name="python")
# Load all languages and concatenate
from datasets import concatenate_datasets
all_langs = concatenate_datasets([
load_dataset("minishlab/tokenlearn-cornstack-docs-coderankembed", name=lang)["train"]
for lang in ["python", "java", "php", "go", "javascript", "ruby"]
])
Creation
Featurized from CornStack using nomic-ai/CodeRankEmbed with mean token pooling (BOS/EOS excluded). Two sampling seeds (42 and 100) were used with a 10k streaming shuffle buffer to maximise diversity. Texts are truncated to 512 tokens.
Library Authors
Tokenlearn was developed by the Minish team consisting of Stephan Tulkens and Thomas van Dongen.
Citation
@software{minishlab2024model2vec,
author = {Stephan Tulkens and {van Dongen}, Thomas},
title = {Model2Vec: Fast State-of-the-Art Static Embeddings},
year = {2024},
publisher = {Zenodo},
doi = {10.5281/zenodo.17270888},
url = {https://github.com/MinishLab/model2vec},
license = {MIT}
}
- Downloads last month
- 28