https://arweave.net/EnGDZNaF-EcV4JACv3pU17bTIxXfiaN7uC7SCrVDe7Q#minblockheight=1147088&minblockhash=eJ-R6Zx9SV2Bir_PWTm_20hFj2sG cOUwZhm50BRuYVeHOORip92wVbXPpCriR65E {"time": "2023-03-28T08:39:26.500652", "lm": "openai", "model": "gpt-4", "chat": true, "kwparams": {"temperature": 0, "top_p": 0}, "input": "\n initial commit\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\nnew file mode 100644\n--- /dev/null\n+++ b/lm/__main__.py\n@@ -0,0 +1,44 @@\n+from .lm_spellbook import spellbook\n+\n+import argparse\n+import asyncio\n+import datetime\n+import json\n+import os\n+import sys\n+\n+async def main():\n+ timestamp = datetime.datetime.now().isoformat()\n+ parser = argparse.ArgumentParser()\n+ parser.add_argument('-l', '--logfile', type=argparse.FileType('w'), default=f'{timestamp}.json', help='Log file')\n+ parser.add_argument('-o', '--output', type=argparse.FileType('w'), default=sys.stdout, help='Output file')\n+ parser.add_argument('input_data', nargs='*', help='Input data or filename')\n+ args = parser.parse_args()\n+\n+ if len(args.input_data) == 1 and os.path.exists(args.input_data[0]):\n+ with open(args.input_data[1], 'r') as f:\n+ input_data = f.read()\n+ log_input(input_data, args.output)\n+ elif args.input_data:\n+ input_data = ' '.join(args.input_data)\n+ else:\n+ input_data = sys.stdin.read()\n+\n+ api = spellbook(\n+ api_key = 'clfcfnj7m003xti1a7hxt1739',\n+ app_id = 'sm43s5r'\n+ )\n+ async with api:\n+ response = await api(input_data)\n+\n+ json.dump({\n+ 'time': timestamp,\n+ 'url': api.url,\n+ 'input': input_data,\n+ 'output': response,\n+ }, args.logfile)\n+\n+ args.output.write(response)\n+\n+if __name__ == \"__main__\":\n+ asyncio.run(main())\n\n factored logging into log_jsonfiles.py\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -1,4 +1,5 @@\n from .lm_spellbook import spellbook\n+from .log_jsonfiles import jsonfiles\n \n import argparse\n import asyncio\n@@ -10,7 +11,6 @@ import sys\n async def main():\n timestamp = datetime.datetime.now().isoformat()\n parser = argparse.ArgumentParser()\n- parser.add_argument('-l', '--logfile', type=argparse.FileType('w'), default=f'{timestamp}.json', help='Log file')\n parser.add_argument('-o', '--output', type=argparse.FileType('w'), default=sys.stdout, help='Output file')\n parser.add_argument('input_data', nargs='*', help='Input data or filename')\n args = parser.parse_args()\n@@ -28,15 +28,16 @@ async def main():\n api_key = 'clfcfnj7m003xti1a7hxt1739',\n app_id = 'sm43s5r'\n )\n- async with api:\n+ log = jsonfiles(prefix = 'gpt4-')\n+ \n+ async with api, log:\n response = await api(input_data)\n-\n- json.dump({\n- 'time': timestamp,\n- 'url': api.url,\n- 'input': input_data,\n- 'output': response,\n- }, args.logfile)\n+ await log(\n+ time = timestamp,\n+ url = api.url,\n+ input = input_data,\n+ output = response,\n+ )\n \n args.output.write(response)\n \n\n install console script; fix duplicate entry\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -8,7 +8,7 @@ import json\n import os\n import sys\n \n-async def main():\n+async def amain():\n timestamp = datetime.datetime.now().isoformat()\n parser = argparse.ArgumentParser()\n parser.add_argument('-o', '--output', type=argparse.FileType('w'), default=sys.stdout, help='Output file')\n@@ -41,5 +41,8 @@ async def main():\n \n args.output.write(response)\n \n+def main():\n+ asyncio.run(amain())\n+\n if __name__ == \"__main__\":\n- asyncio.run(main())\n+ main()\n\n lm/lm and lm/log subpackages\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -1,5 +1,5 @@\n-from .lm_spellbook import spellbook\n-from .log_jsonfiles import jsonfiles\n+from .lm.spellbook import spellbook\n+from .log.jsonfiles import jsonfiles\n \n import argparse\n import asyncio\n\n added openai api, untested\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -1,4 +1,5 @@\n from .lm.spellbook import spellbook\n+from .lm.openai import openai\n from .log.jsonfiles import jsonfiles\n \n import argparse\n@@ -24,17 +25,14 @@ async def amain():\n else:\n input_data = sys.stdin.read()\n \n- api = spellbook(\n- api_key = 'clfcfnj7m003xti1a7hxt1739',\n- app_id = 'sm43s5r'\n- )\n+ api = spellbook()\n log = jsonfiles(prefix = 'gpt4-')\n \n async with api, log:\n response = await api(input_data)\n await log(\n time = timestamp,\n- url = api.url,\n+ **api.metadata,\n input = input_data,\n output = response,\n )\n\n read content of filename parameters\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -6,6 +6,7 @@ import argparse\n import asyncio\n import datetime\n import json\n+import logging\n import os\n import sys\n \n@@ -16,12 +17,21 @@ async def amain():\n parser.add_argument('input_data', nargs='*', help='Input data or filename')\n args = parser.parse_args()\n \n- if len(args.input_data) == 1 and os.path.exists(args.input_data[0]):\n- with open(args.input_data[1], 'r') as f:\n- input_data = f.read()\n- log_input(input_data, args.output)\n- elif args.input_data:\n- input_data = ' '.join(args.input_data)\n+ input_data = []\n+ logger = logging.getLogger(__name__)\n+ for item in args.input_data:\n+ if os.path.exists(item):\n+ logger.warn(f'Reading content from file \"{item}\"')\n+ with open(item, 'r') as f:\n+ if input_data:\n+ input_data[-1] = '\\n'\n+ input_data.extend((f.read(), '\\n'))\n+ else:\n+ input_data.extend((item, ' '))\n+\n+ if input_data:\n+ input_data.pop() # trailing whitespace\n+ input_data = ''.join(input_data)\n else:\n input_data = sys.stdin.read()\n \n\n committing current work\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -35,6 +35,7 @@ async def amain():\n else:\n input_data = sys.stdin.read()\n \n+ #api = openai(model = 'gpt-4')\n api = spellbook()\n log = jsonfiles(prefix = 'gpt4-')\n \n\n log to arweave\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -1,6 +1,7 @@\n from .lm.spellbook import spellbook\n from .lm.openai import openai\n from .log.jsonfiles import jsonfiles\n+from .log.arweaveditems import arweaveditems\n \n import argparse\n import asyncio\n@@ -38,15 +39,25 @@ async def amain():\n #api = openai(model = 'gpt-4')\n api = spellbook()\n log = jsonfiles(prefix = 'gpt4-')\n+ log2 = arweaveditems()\n \n- async with api, log:\n+ async with api, log, log2:\n response = await api(input_data)\n- await log(\n- time = timestamp,\n- **api.metadata,\n- input = input_data,\n- output = response,\n- )\n+ for addr in await asyncio.gather(\n+ log(\n+ time = timestamp,\n+ **api.metadata,\n+ input = input_data,\n+ output = response,\n+ ),\n+ log2(\n+ time = timestamp,\n+ **api.metadata,\n+ input = input_data,\n+ output = response,\n+ )\n+ ):\n+ logger.warn(addr)\n \n args.output.write(response)\n \n\n added __init__.py for subpackages\n\ndiff --git a/lm/lm/__init__.py b/lm/lm/__init__.py\nnew file mode 100644\n--- /dev/null\n+++ b/lm/lm/__init__.py\ndiff --git a/lm/log/__init__.py b/lm/log/__init__.py\nnew file mode 100644\n--- /dev/null\n+++ b/lm/log/__init__.py\n\n enable openai, i have not gotten llama going locally yet\n\ndiff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -1,4 +1,4 @@\n-from .lm.spellbook import spellbook\n+from .lm.gradio import gradio\n from .lm.openai import openai\n from .log.jsonfiles import jsonfiles\n from .log.arweaveditems import arweaveditems\n@@ -36,8 +36,8 @@ async def amain():\n else:\n input_data = sys.stdin.read()\n \n- #api = openai(model = 'gpt-4')\n- api = spellbook()\n+ api = openai(model = 'gpt-4')\n+ #api = gradio()\n log = jsonfiles(prefix = 'gpt4-')\n log2 = arweaveditems()\n \n\n added commandline options for selecting any lm.lm module and lm.log module\n\n", "output": "diff --git a/lm/__main__.py b/lm/__main__.py\n--- a/lm/__main__.py\n+++ b/lm/__main__.py\n@@ -1,4 +1,4 @@\n-from .lm.spellbook import spellbook\n+from .lm import spellbook, openai, gradio\n from .log.jsonfiles import jsonfiles\n from .log.arweaveditems import arweaveditems\n \n@@ -9,6 +9,12 @@ import sys\n async def amain():\n timestamp = datetime.datetime.now().isoformat()\n parser = argparse.ArgumentParser()\n+ parser.add_argument('--lm', choices=['spellbook', 'openai', 'gradio'], default='spellbook', help='Language model to use')\n+ parser.add_argument('--log', choices=['jsonfiles', 'arweaveditems'], default='jsonfiles', help='Logging method to use')\n parser.add_argument('-o', '--output', type=argparse.FileType('w'), default=sys.stdout, help='Output file')\n parser.add_argument('input_data', nargs='*', help='Input data or filename')\n args = parser.parse_args()\n@@ -36,8 +42,12 @@ async def amain():\n else:\n input_data = sys.stdin.read()\n \n- api = spellbook()\n- log = jsonfiles(prefix = 'gpt4-')\n+ api = {\n+ 'spellbook': spellbook,\n+ 'openai': openai,\n+ 'gradio': gradio,\n+ }[args.lm]()\n+ log = {'jsonfiles': jsonfiles, 'arweaveditems': arweaveditems}[args.log](prefix='gpt4-')\n log2 = arweaveditems()\n \n async with api, log, log2:"}