diff --git a/package-lock.json b/package-lock.json index 5edc1ca..d799cb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "opencommit", - "version": "1.1.22", + "version": "1.1.24", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opencommit", - "version": "1.1.22", + "version": "1.1.24", "license": "MIT", "dependencies": { "@clack/prompts": "^0.6.1", + "@dqbd/tiktoken": "^1.0.2", "axios": "^1.3.4", "chalk": "^5.2.0", "cleye": "^1.3.2", @@ -83,6 +84,11 @@ "node": ">=12" } }, + "node_modules/@dqbd/tiktoken": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.2.tgz", + "integrity": "sha512-AjGTBRWsMoVmVeN55NLyupyM8TNamOUBl6tj5t/leLDVup3CFGO9tVagNL1jf3GyZLkWZSTmYVbPQ/M2LEcNzw==" + }, "node_modules/@esbuild/android-arm": { "version": "0.15.18", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.18.tgz", diff --git a/package.json b/package.json index 62505d8..0ce4593 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opencommit", - "version": "1.1.22", + "version": "1.1.24", "description": "GPT CLI to auto-generate impressive commits in 1 second. Killing lame commits with AI đŸ€ŻđŸ”«", "keywords": [ "git", @@ -60,6 +60,7 @@ }, "dependencies": { "@clack/prompts": "^0.6.1", + "@dqbd/tiktoken": "^1.0.2", "axios": "^1.3.4", "chalk": "^5.2.0", "cleye": "^1.3.2", diff --git a/src/generateCommitMessageFromGitDiff.ts b/src/generateCommitMessageFromGitDiff.ts index d3f820a..992b1e1 100644 --- a/src/generateCommitMessageFromGitDiff.ts +++ b/src/generateCommitMessageFromGitDiff.ts @@ -6,6 +6,7 @@ import { api } from './api'; import { getConfig } from './commands/config'; import { mergeStrings } from './utils/mergeStrings'; import { i18n, I18nLocals } from './i18n'; +import { tokenCount } from './utils/tokenCount'; const config = getConfig(); const translation = i18n[(config?.language as I18nLocals) || 'en']; @@ -22,7 +23,10 @@ const INIT_MESSAGES_PROMPT: Array = [ config?.description ? 'Add a short description of why the commit is done after the commit message. Don\'t start it with "This commit", just describe the changes' : "Don't add any descriptions to the commit, only commit message" - }. Use the present tense. Use ${translation.localLanguage} to answer.` + }. + Use the present tense. + Lines must not be longer 74 characters. + Use ${translation.localLanguage} to answer.` }, { role: ChatCompletionRequestMessageRoleEnum.User, @@ -81,8 +85,8 @@ interface GenerateCommitMessageError { } const INIT_MESSAGES_PROMPT_LENGTH = INIT_MESSAGES_PROMPT.map( - (msg) => msg.content -).join('').length; + (msg) => tokenCount(msg.content) + 4 +).reduce((a, b) => a + b, 0); const MAX_REQ_TOKENS = 3900 - INIT_MESSAGES_PROMPT_LENGTH; @@ -90,7 +94,7 @@ export const generateCommitMessageWithChatCompletion = async ( diff: string ): Promise => { try { - if (diff.length >= MAX_REQ_TOKENS) { + if (tokenCount(diff) >= MAX_REQ_TOKENS) { const commitMessagePromises = getCommitMsgsPromisesFromFileDiffs(diff); const commitMessages = await Promise.all(commitMessagePromises); @@ -145,7 +149,7 @@ function getCommitMsgsPromisesFromFileDiffs(diff: string) { const commitMessagePromises = []; for (const fileDiff of mergedFilesDiffs) { - if (fileDiff.length >= MAX_REQ_TOKENS) { + if (tokenCount(fileDiff) >= MAX_REQ_TOKENS) { // if file-diff is bigger than gpt context — split fileDiff into lineDiff const messagesPromises = getMessagesPromisesByLines(fileDiff, separator); diff --git a/src/i18n/index.ts b/src/i18n/index.ts index 53594c4..079e05e 100644 --- a/src/i18n/index.ts +++ b/src/i18n/index.ts @@ -9,6 +9,7 @@ import ja from '../i18n/ja.json' assert { type: 'json' }; import pt_br from '../i18n/pt_br.json' assert { type: 'json' }; import vi_VN from '../i18n/vi_VN.json' assert { type: 'json' }; import es_ES from '../i18n/es_ES.json' assert { type: 'json' }; +import sv from '../i18n/sv.json' assert { type: 'json' }; export enum I18nLocals { 'en' = 'en', @@ -20,7 +21,8 @@ export enum I18nLocals { 'it' = 'it', 'ko' = 'ko', 'pt_br' = 'pt_br', - 'es_ES' = 'es_ES' + 'es_ES' = 'es_ES', + 'sv' = 'sv', }; export const i18n = { @@ -34,7 +36,8 @@ export const i18n = { ko, pt_br, vi_VN, - es_ES + es_ES, + sv }; export const I18N_CONFIG_ALIAS: { [key: string]: string[] } = { @@ -49,6 +52,7 @@ export const I18N_CONFIG_ALIAS: { [key: string]: string[] } = { vi_VN: ['vi_VN', 'Vietnamese', 'tiáșżng Việt'], en: ['en', 'English', 'english'], es_ES: ['es_ES', 'Spanish', 'español'], + sv: ['sv', 'Swedish', 'Svenska'], }; export function getI18nLocal(value: string): string | boolean { diff --git a/src/i18n/sv.json b/src/i18n/sv.json new file mode 100644 index 0000000..eb86aee --- /dev/null +++ b/src/i18n/sv.json @@ -0,0 +1,6 @@ +{ +"localLanguage": "svenska", +"commitFix": "fixa(server.ts): Ă€ndra variabelnamnet för port frĂ„n smĂ„ bokstĂ€ver till stora bokstĂ€ver PORT", +"commitFeat": "nyhet(server.ts): lĂ€gg till stöd för process.env.PORT miljövariabel", +"commitDescription": "Variabeln som innehĂ„ller portnumret heter nu PORT vilket förbĂ€ttrar konsekvensen med namngivningskonventionerna eftersom PORT Ă€r en konstant. Stöd för en miljövariabel gör att applikationen kan vara mer flexibel dĂ„ den nu kan köras pĂ„ vilken port som helst som specificeras via miljövariabeln process.env.PORT." +} \ No newline at end of file diff --git a/src/utils/mergeStrings.ts b/src/utils/mergeStrings.ts index 7b55a99..ee35f50 100644 --- a/src/utils/mergeStrings.ts +++ b/src/utils/mergeStrings.ts @@ -1,8 +1,9 @@ +import { tokenCount } from './tokenCount' export function mergeStrings(arr: string[], maxStringLength: number): string[] { const mergedArr: string[] = []; let currentItem: string = arr[0]; for (const item of arr.slice(1)) { - if (currentItem.length + item.length <= maxStringLength) { + if (tokenCount(currentItem + item) <= maxStringLength) { currentItem += item; } else { mergedArr.push(currentItem); diff --git a/src/utils/tokenCount.ts b/src/utils/tokenCount.ts new file mode 100644 index 0000000..84e4f23 --- /dev/null +++ b/src/utils/tokenCount.ts @@ -0,0 +1,14 @@ +import { Tiktoken } from "@dqbd/tiktoken/lite" +import cl100k_base from "@dqbd/tiktoken/encoders/cl100k_base.json" assert{type: "json"} + +export function tokenCount(content: string): number { + const encoding = new Tiktoken( + cl100k_base.bpe_ranks, + cl100k_base.special_tokens, + cl100k_base.pat_str + ); + const tokens = encoding.encode(content); + encoding.free(); + + return tokens.length; +}